You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2017/11/18 03:31:22 UTC

[JENKINS] Lucene-Solr-7.x-Linux (32bit/jdk1.8.0_144) - Build # 845 - Still Unstable!

Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Linux/845/
Java: 32bit/jdk1.8.0_144 -server -XX:+UseConcMarkSweepGC

3 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.core.BlobRepositoryCloudTest

Error Message:
Error from server at https://127.0.0.1:35749/solr: create the collection time out:180s

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:35749/solr: create the collection time out:180s
	at __randomizedtesting.SeedInfo.seed([161D53EED566A556]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1104)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:883)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:816)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
	at org.apache.solr.core.BlobRepositoryCloudTest.setupCluster(BlobRepositoryCloudTest.java:51)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:874)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.LeaderFailureAfterFreshStartTest.test

Error Message:
Timeout occured while waiting response from server at: https://127.0.0.1:34365/sb_s

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:34365/sb_s
	at __randomizedtesting.SeedInfo.seed([161D53EED566A556:9E496C347B9AC8AE]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:654)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1104)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:883)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:816)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:314)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:991)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:171)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
	at sun.security.ssl.InputRecord.read(InputRecord.java:503)
	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:983)
	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:940)
	at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:542)
	... 43 more


FAILED:  org.apache.solr.schema.TestCloudManagedSchema.test

Error Message:
Timeout occured while waiting response from server at: https://127.0.0.1:36553/ozsr/e

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:36553/ozsr/e
	at __randomizedtesting.SeedInfo.seed([161D53EED566A556:9E496C347B9AC8AE]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:654)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1104)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:883)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:816)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:314)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:991)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:171)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
	at sun.security.ssl.InputRecord.read(InputRecord.java:503)
	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:983)
	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:940)
	at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:542)
	... 43 more




Build Log:
[...truncated 12308 lines...]
   [junit4] Suite: org.apache.solr.core.BlobRepositoryCloudTest
   [junit4]   2> Creating dataDir: /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.core.BlobRepositoryCloudTest_161D53EED566A556-001/init-core-data-001
   [junit4]   2> 545519 INFO  (SUITE-BlobRepositoryCloudTest-seed#[161D53EED566A556]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 545520 INFO  (SUITE-BlobRepositoryCloudTest-seed#[161D53EED566A556]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 545521 INFO  (SUITE-BlobRepositoryCloudTest-seed#[161D53EED566A556]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 545521 INFO  (SUITE-BlobRepositoryCloudTest-seed#[161D53EED566A556]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 1 servers in /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.core.BlobRepositoryCloudTest_161D53EED566A556-001/tempDir-001
   [junit4]   2> 545522 INFO  (SUITE-BlobRepositoryCloudTest-seed#[161D53EED566A556]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 545525 INFO  (Thread-709) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 545525 INFO  (Thread-709) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 545557 ERROR (Thread-709) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 545622 INFO  (SUITE-BlobRepositoryCloudTest-seed#[161D53EED566A556]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:44043
   [junit4]   2> 545642 INFO  (jetty-launcher-652-thread-1) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 545685 INFO  (jetty-launcher-652-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@a59a56{/solr,null,AVAILABLE}
   [junit4]   2> 545687 INFO  (jetty-launcher-652-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@70b4dc{SSL,[ssl, http/1.1]}{127.0.0.1:35749}
   [junit4]   2> 545687 INFO  (jetty-launcher-652-thread-1) [    ] o.e.j.s.Server Started @547052ms
   [junit4]   2> 545687 INFO  (jetty-launcher-652-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=35749}
   [junit4]   2> 545687 ERROR (jetty-launcher-652-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 545687 INFO  (jetty-launcher-652-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.2.0
   [junit4]   2> 545687 INFO  (jetty-launcher-652-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 545687 INFO  (jetty-launcher-652-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 545687 INFO  (jetty-launcher-652-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-11-18T02:29:14.215Z
   [junit4]   2> 545700 INFO  (jetty-launcher-652-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 545722 INFO  (jetty-launcher-652-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:44043/solr
   [junit4]   2> 545916 INFO  (jetty-launcher-652-thread-1) [n:127.0.0.1:35749_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 545917 INFO  (jetty-launcher-652-thread-1) [n:127.0.0.1:35749_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:35749_solr
   [junit4]   2> 545917 INFO  (jetty-launcher-652-thread-1) [n:127.0.0.1:35749_solr    ] o.a.s.c.Overseer Overseer (id=99023071088345091-127.0.0.1:35749_solr-n_0000000000) starting
   [junit4]   2> 545949 INFO  (jetty-launcher-652-thread-1) [n:127.0.0.1:35749_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35749_solr
   [junit4]   2> 545950 INFO  (zkCallback-659-thread-1-processing-n:127.0.0.1:35749_solr) [n:127.0.0.1:35749_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 546093 INFO  (jetty-launcher-652-thread-1) [n:127.0.0.1:35749_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35749.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@889af
   [junit4]   2> 546130 INFO  (jetty-launcher-652-thread-1) [n:127.0.0.1:35749_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35749.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@889af
   [junit4]   2> 546130 INFO  (jetty-launcher-652-thread-1) [n:127.0.0.1:35749_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35749.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@889af
   [junit4]   2> 546131 INFO  (jetty-launcher-652-thread-1) [n:127.0.0.1:35749_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.core.BlobRepositoryCloudTest_161D53EED566A556-001/tempDir-001/node1/.
   [junit4]   2> 546203 INFO  (SUITE-BlobRepositoryCloudTest-seed#[161D53EED566A556]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 546204 INFO  (SUITE-BlobRepositoryCloudTest-seed#[161D53EED566A556]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:44043/solr ready
   [junit4]   2> 546293 INFO  (qtp15676671-3463) [n:127.0.0.1:35749_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&name=.system&nrtReplicas=1&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 546305 INFO  (OverseerThreadFactory-1736-thread-1-processing-n:127.0.0.1:35749_solr) [n:127.0.0.1:35749_solr    ] o.a.s.c.CreateCollectionCmd Create collection .system
   [junit4]   2> 546415 INFO  (OverseerStateUpdate-99023071088345091-127.0.0.1:35749_solr-n_0000000000) [n:127.0.0.1:35749_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":".system",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":".system_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:35749/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 546638 INFO  (qtp15676671-3462) [n:127.0.0.1:35749_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node2&collection.configName=.system&newCollection=true&name=.system_shard1_replica_n1&action=CREATE&numShards=1&collection=.system&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 546638 INFO  (qtp15676671-3462) [n:127.0.0.1:35749_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 546750 INFO  (zkCallback-659-thread-1-processing-n:127.0.0.1:35749_solr) [n:127.0.0.1:35749_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 546751 INFO  (zkCallback-659-thread-2-processing-n:127.0.0.1:35749_solr) [n:127.0.0.1:35749_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 547662 INFO  (qtp15676671-3462) [n:127.0.0.1:35749_solr c:.system s:shard1 r:core_node2 x:.system_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
   [junit4]   2> 547665 INFO  (qtp15676671-3462) [n:127.0.0.1:35749_solr c:.system s:shard1 r:core_node2 x:.system_shard1_replica_n1] o.a.s.s.ManagedIndexSchemaFactory The schema is configured as managed, but managed schema resource managed-schema not found - loading non-managed schema schema.xml instead
   [junit4]   2> 547666 INFO  (qtp15676671-3462) [n:127.0.0.1:35749_solr c:.system s:shard1 r:core_node2 x:.system_shard1_replica_n1] o.a.s.s.IndexSchema [.system_shard1_replica_n1] Schema name=_system collection or core
   [junit4]   2> 547670 INFO  (qtp15676671-3462) [n:127.0.0.1:35749_solr c:.system s:shard1 r:core_node2 x:.system_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema _system collection or core/1.1 with uniqueid field id
   [junit4]   2> 547672 INFO  (qtp15676671-3462) [n:127.0.0.1:35749_solr c:.system s:shard1 r:core_node2 x:.system_shard1_replica_n1] o.a.s.s.ManagedIndexSchema Created and persisted managed schema znode at /configs/.system/managed-schema
   [junit4]   2> 547698 INFO  (qtp15676671-3462) [n:127.0.0.1:35749_solr c:.system s:shard1 r:core_node2 x:.system_shard1_replica_n1] o.a.s.s.ManagedIndexSchemaFactory After upgrading to managed schema in ZooKeeper, renamed the non-managed schema /configs/.system/schema.xml to /configs/.system/schema.xml.bak
   [junit4]   2> 547698 INFO  (qtp15676671-3462) [n:127.0.0.1:35749_solr c:.system s:shard1 r:core_node2 x:.system_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore '.system_shard1_replica_n1' using configuration from collection .system, trusted=true
   [junit4]   2> 547700 INFO  (qtp15676671-3462) [n:127.0.0.1:35749_solr c:.system s:shard1 r:core_node2 x:.system_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35749.solr.core..system.shard1.replica_n1' (registry 'solr.core..system.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@889af
   [junit4]   2> 547700 INFO  (qtp15676671-3462) [n:127.0.0.1:35749_solr c:.system s:shard1 r:core_node2 x:.system_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 547700 INFO  (qtp15676671-3462) [n:127.0.0.1:35749_solr c:.system s:shard1 r:core_node2 x:.system_shard1_replica_n1] o.a.s.c.SolrCore [[.system_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.core.BlobRepositoryCloudTest_161D53EED566A556-001/tempDir-001/node1/.system_shard1_replica_n1], dataDir=[/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.core.BlobRepositoryCloudTest_161D53EED566A556-001/tempDir-001/node1/./.system_shard1_replica_n1/data/]
   [junit4]   2> 547776 INFO  (qtp15676671-3462) [n:127.0.0.1:35749_solr c:.system s:shard1 r:core_node2 x:.system_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 547776 INFO  (qtp15676671-3462) [n:127.0.0.1:35749_solr c:.system s:shard1 r:core_node2 x:.system_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 547778 INFO  (qtp15676671-3462) [n:127.0.0.1:35749_solr c:.system s:shard1 r:core_node2 x:.system_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: if 1 uncommited docs 
   [junit4]   2> 547778 INFO  (qtp15676671-3462) [n:127.0.0.1:35749_solr c:.system s:shard1 r:core_node2 x:.system_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 547794 INFO  (qtp15676671-3462) [n:127.0.0.1:35749_solr c:.system s:shard1 r:core_node2 x:.system_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@1de3be8[.system_shard1_replica_n1] main]
   [junit4]   2> 547806 INFO  (qtp15676671-3462) [n:127.0.0.1:35749_solr c:.system s:shard1 r:core_node2 x:.system_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/.system
   [junit4]   2> 547806 INFO  (qtp15676671-3462) [n:127.0.0.1:35749_solr c:.system s:shard1 r:core_node2 x:.system_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/.system
   [junit4]   2> 547806 INFO  (qtp15676671-3462) [n:127.0.0.1:35749_solr c:.system s:shard1 r:core_node2 x:.system_shard1_replica_n1] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/.system/managed-schema
   [junit4]   2> 547810 INFO  (qtp15676671-3462) [n:127.0.0.1:35749_solr c:.system s:shard1 r:core_node2 x:.system_shard1_replica_n1] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
   [junit4]   2> 547811 INFO  (qtp15676671-3462) [n:127.0.0.1:35749_solr c:.system s:shard1 r:core_node2 x:.system_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 547811 INFO  (searcherExecutor-1739-thread-1-processing-n:127.0.0.1:35749_solr x:.system_shard1_replica_n1 s:shard1 c:.system r:core_node2) [n:127.0.0.1:35749_solr c:.system s:shard1 r:core_node2 x:.system_shard1_replica_n1] o.a.s.c.SolrCore [.system_shard1_replica_n1] Registered new searcher Searcher@1de3be8[.system_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 547812 INFO  (qtp15676671-3462) [n:127.0.0.1:35749_solr c:.system s:shard1 r:core_node2 x:.system_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1584369139806371840
   [junit4]   2> 547828 INFO  (qtp15676671-3462) [n:127.0.0.1:35749_solr c:.system s:shard1 r:core_node2 x:.system_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 547828 INFO  (qtp15676671-3462) [n:127.0.0.1:35749_solr c:.system s:shard1 r:core_node2 x:.system_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 547828 INFO  (qtp15676671-3462) [n:127.0.0.1:35749_solr c:.system s:shard1 r:core_node2 x:.system_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:35749/solr/.system_shard1_replica_n1/
   [junit4]   2> 547828 INFO  (qtp15676671-3462) [n:127.0.0.1:35749_solr c:.system s:shard1 r:core_node2 x:.system_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 547828 INFO  (qtp15676671-3462) [n:127.0.0.1:35749_solr c:.system s:shard1 r:core_node2 x:.system_shard1_replica_n1] o.a.s.c.SyncStrategy https://127.0.0.1:35749/solr/.system_shard1_replica_n1/ has no replicas
   [junit4]   2> 547828 INFO  (qtp15676671-3462) [n:127.0.0.1:35749_solr c:.system s:shard1 r:core_node2 x:.system_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 547863 INFO  (qtp15676671-3462) [n:127.0.0.1:35749_solr c:.system s:shard1 r:core_node2 x:.system_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:35749/solr/.system_shard1_replica_n1/ shard1
   [junit4]   2> 726300 ERROR (qtp15676671-3463) [n:127.0.0.1:35749_solr    ] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: create the collection time out:180s
   [junit4]   2> 	at org.apache.solr.handler.admin.CollectionsHandler.handleResponse(CollectionsHandler.java:312)
   [junit4]   2> 	at org.apache.solr.handler.admin.CollectionsHandler.invokeAction(CollectionsHandler.java:244)
   [junit4]   2> 	at org.apache.solr.handler.admin.CollectionsHandler.handleRequestBody(CollectionsHandler.java:222)
   [junit4]   2> 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:735)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:716)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:497)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:493)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 
   [junit4]   2> 726300 INFO  (qtp15676671-3463) [n:127.0.0.1:35749_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&name=.system&nrtReplicas=1&action=CREATE&numShards=1&wt=javabin&version=2} status=500 QTime=180006
   [junit4]   2> 726300 ERROR (qtp15676671-3463) [n:127.0.0.1:35749_solr    ] o.a.s.s.HttpSolrCall null:org.apache.solr.common.SolrException: create the collection time out:180s
   [junit4]   2> 	at org.apache.solr.handler.admin.CollectionsHandler.handleResponse(CollectionsHandler.java:312)
   [junit4]   2> 	at org.apache.solr.handler.admin.CollectionsHandler.invokeAction(CollectionsHandler.java:244)
   [junit4]   2> 	at org.apache.solr.handler.admin.CollectionsHandler.handleRequestBody(CollectionsHandler.java:222)
   [junit4]   2> 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:735)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:716)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:497)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:493)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 
   [junit4]   2> 726302 INFO  (jetty-closer-653-thread-1) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@70b4dc{SSL,[ssl, http/1.1]}{127.0.0.1:0}
   [junit4]   2> 726302 INFO  (jetty-closer-653-thread-1) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=15600011
   [junit4]   2> 726302 INFO  (jetty-closer-653-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 726302 INFO  (jetty-closer-653-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@2a1403: rootName = solr_35749, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@55bacf
   [junit4]   2> 726304 ERROR (OverseerThreadFactory-1736-thread-1-processing-n:127.0.0.1:35749_solr) [n:127.0.0.1:35749_solr    ] o.a.s.c.OverseerCollectionMessageHandler Error from shard: https://127.0.0.1:35749/solr
   [junit4]   2> org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: https://127.0.0.1:35749/solr
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:657)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
   [junit4]   2> 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
   [junit4]   2> 	at org.apache.solr.handler.component.HttpShardHandler.lambda$submit$0(HttpShardHandler.java:172)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:35749 failed to respond
   [junit4]   2> 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:141)
   [junit4]   2> 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
   [junit4]   2> 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
   [junit4]   2> 	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
   [junit4]   2> 	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
   [junit4]   2> 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
   [junit4]   2> 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
   [junit4]   2> 	at org.apache.solr.util.stats.InstrumentedHttpRequestExecutor.execute(InstrumentedHttpRequestExecutor.java:118)
   [junit4]   2> 	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
   [junit4]   2> 	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
   [junit4]   2> 	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
   [junit4]   2> 	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
   [junit4]   2> 	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
   [junit4]   2> 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
   [junit4]   2> 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:542)
   [junit4]   2> 	... 12 more
   [junit4]   2> 726305 ERROR (OverseerThreadFactory-1736-thread-1-processing-n:127.0.0.1:35749_solr) [n:127.0.0.1:35749_solr    ] o.a.s.c.OverseerCollectionMessageHandler Cleaning up collection [.system].
   [junit4]   2> 726305 INFO  (OverseerThreadFactory-1736-thread-1-processing-n:127.0.0.1:35749_solr) [n:127.0.0.1:35749_solr    ] o.a.s.c.OverseerCollectionMessageHandler Executing Collection Cmd : action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true
   [junit4]   2> 726306 ERROR (OverseerThreadFactory-1736-thread-1-processing-n:127.0.0.1:35749_solr) [n:127.0.0.1:35749_solr    ] o.a.s.c.OverseerCollectionMessageHandler Error from shard: https://127.0.0.1:35749/solr
   [junit4]   2> org.apache.solr.client.solrj.SolrServerException: Server refused connection at: https://127.0.0.1:35749/solr
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:650)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
   [junit4]   2> 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
   [junit4]   2> 	at org.apache.solr.handler.component.HttpShardHandler.lambda$submit$0(HttpShardHandler.java:172)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: org.apache.http.conn.HttpHostConnectException: Connect to 127.0.0.1:35749 [/127.0.0.1] failed: Connection refused (Connection refused)
   [junit4]   2> 	at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:159)
   [junit4]   2> 	at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:359)
   [junit4]   2> 	at org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:381)
   [junit4]   2> 	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:237)
   [junit4]   2> 	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
   [junit4]   2> 	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
   [junit4]   2> 	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
   [junit4]   2> 	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
   [junit4]   2> 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
   [junit4]   2> 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:542)
   [junit4]   2> 	... 12 more
   [junit4]   2> Caused by: java.net.ConnectException: Connection refused (Connection refused)
   [junit4]   2> 	at java.net.PlainSocketImpl.socketConnect(Native Method)
   [junit4]   2> 	at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
   [junit4]   2> 	at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
   [junit4]   2> 	at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
   [junit4]   2> 	at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
   [junit4]   2> 	at java.net.Socket.connect(Socket.java:589)
   [junit4]   2> 	at org.apache.http.conn.ssl.SSLConnectionSocketFactory.connectSocket(SSLConnectionSocketFactory.java:339)
   [junit4]   2> 	at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:142)
   [junit4]   2> 	... 22 more
   [junit4]   2> 726309 INFO  (jetty-closer-653-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 726309 INFO  (jetty-closer-653-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@150a123: rootName = solr_35749, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@2d710c
   [junit4]   2> 726314 INFO  (jetty-closer-653-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 726314 INFO  (jetty-closer-653-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@1845097: rootName = solr_35749, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@1b8bea7
   [junit4]   2> 726315 INFO  (jetty-closer-653-thread-1) [    ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:35749_solr
   [junit4]   2> 726315 INFO  (jetty-closer-653-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 726318 INFO  (coreCloseExecutor-1744-thread-1) [n:127.0.0.1:35749_solr c:.system s:shard1 r:core_node2 x:.system_shard1_replica_n1] o.a.s.c.SolrCore [.system_shard1_replica_n1]  CLOSING SolrCore org.apache.solr.core.SolrCore@72d5d2
   [junit4]   2> 726318 INFO  (coreCloseExecutor-1744-thread-1) [n:127.0.0.1:35749_solr c:.system s:shard1 r:core_node2 x:.system_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core..system.shard1.replica_n1, tag=7525842
   [junit4]   2> 726318 INFO  (coreCloseExecutor-1744-thread-1) [n:127.0.0.1:35749_solr c:.system s:shard1 r:core_node2 x:.system_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@a54699: rootName = solr_35749, domain = solr.core..system.shard1.replica_n1, service url = null, agent id = null] for registry solr.core..system.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@2d5667
   [junit4]   2> 726319 INFO  (zkCallback-659-thread-5-processing-n:127.0.0.1:35749_solr) [n:127.0.0.1:35749_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
   [junit4]   2> 726326 INFO  (coreCloseExecutor-1744-thread-1) [n:127.0.0.1:35749_solr c:.system s:shard1 r:core_node2 x:.system_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection..system.shard1.leader, tag=7525842
   [junit4]   2> 726328 INFO  (jetty-closer-653-thread-1) [    ] o.a.s.c.Overseer Overseer (id=99023071088345091-127.0.0.1:35749_solr-n_0000000000) closing
   [junit4]   2> 726328 INFO  (OverseerStateUpdate-99023071088345091-127.0.0.1:35749_solr-n_0000000000) [n:127.0.0.1:35749_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:35749_solr
   [junit4]   2> 756344 INFO  (zkCallback-659-thread-5-processing-n:127.0.0.1:35749_solr) [n:127.0.0.1:35749_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDeleted path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [0])
   [junit4]   2> 756344 INFO  (zkCallback-659-thread-4-processing-n:127.0.0.1:35749_solr) [n:127.0.0.1:35749_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDeleted path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [0])
   [junit4]   2> 756346 ERROR (OverseerThreadFactory-1736-thread-1-processing-n:127.0.0.1:35749_solr) [n:127.0.0.1:35749_solr    ] o.a.s.c.OverseerCollectionMessageHandler Collection: .system operation: create failed:org.apache.solr.common.SolrException: Could not fully remove collection: .system
   [junit4]   2> 	at org.apache.solr.cloud.DeleteCollectionCmd.call(DeleteCollectionCmd.java:109)
   [junit4]   2> 	at org.apache.solr.cloud.OverseerCollectionMessageHandler.cleanupCollection(OverseerCollectionMessageHandler.java:678)
   [junit4]   2> 	at org.apache.solr.cloud.CreateCollectionCmd.call(CreateCollectionCmd.java:302)
   [junit4]   2> 	at org.apache.solr.cloud.OverseerCollectionMessageHandler.processMessage(OverseerCollectionMessageHandler.java:232)
   [junit4]   2> 	at org.apache.solr.cloud.OverseerTaskProcessor$Runner.run(OverseerTaskProcessor.java:464)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 
   [junit4]   2> 756355 INFO  (OverseerThreadFactory-1736-thread-1-processing-n:127.0.0.1:35749_solr) [n:127.0.0.1:35749_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 756356 WARN  (OverseerAutoScalingTriggerThread-99023071088345091-127.0.0.1:35749_solr-n_0000000000) [n:127.0.0.1:35749_solr    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 756363 INFO  (jetty-closer-653-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@a59a56{/solr,null,UNAVAILABLE}
   [junit4]   2> 756395 ERROR (qtp15676671-3462) [n:127.0.0.1:35749_solr c:.system s:shard1 r:core_node2 x:.system_shard1_replica_n1] o.a.s.c.ZkController Error getting leader from zk
   [junit4]   2> org.apache.solr.common.SolrException: No registered leader was found after waiting for 1220000ms , collection: .system slice: shard1 saw state=null with live_nodes=[]
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:771)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:736)
   [junit4]   2> 	at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1178)
   [junit4]   2> 	at org.apache.solr.cloud.ZkController.register(ZkController.java:1079)
   [junit4]   2> 	at org.apache.solr.cloud.ZkController.register(ZkController.java:1014)
   [junit4]   2> 	at org.apache.solr.core.ZkContainer.lambda$registerInZk$0(ZkContainer.java:181)
   [junit4]   2> 	at org.apache.solr.core.ZkContainer.registerInZk(ZkContainer.java:208)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.registerCore(CoreContainer.java:886)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1047)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:948)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
   [junit4]   2> 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:735)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:716)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:497)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 756396 ERROR (qtp15676671-3462) [n:127.0.0.1:35749_solr c:.system s:shard1 r:core_node2 x:.system_shard1_replica_n1] o.a.s.c.ZkContainer :org.apache.solr.common.SolrException: Error getting leader from zk for shard shard1
   [junit4]   2> 	at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1207)
   [junit4]   2> 	at org.apache.solr.cloud.ZkController.register(ZkController.java:1079)
   [junit4]   2> 	at org.apache.solr.cloud.ZkController.register(ZkController.java:1014)
   [junit4]   2> 	at org.apache.solr.core.ZkContainer.lambda$registerInZk$0(ZkContainer.java:181)
   [junit4]   2> 	at org.apache.solr.core.ZkContainer.registerInZk(ZkContainer.java:208)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.registerCore(CoreContainer.java:886)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1047)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:948)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
   [junit4]   2> 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:735)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:716)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:497)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: No registered leader was found after waiting for 1220000ms , collection: .system slice: shard1 saw state=null with live_nodes=[]
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:771)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:736)
   [junit4]   2> 	at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1178)
   [junit4]   2> 	... 44 more
   [junit4]   2> 
   [junit4]   2> 756397 INFO  (qtp15676671-3462) [n:127.0.0.1:35749_solr c:.system s:shard1 r:core_node2 x:.system_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node2&collection.configName=.system&newCollection=true&name=.system_shard1_replica_n1&action=CREATE&numShards=1&collection=.system&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=209758
   [junit4]   2> 756397 ERROR (SUITE-BlobRepositoryCloudTest-seed#[161D53EED566A556]-worker) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 756398 INFO  (SUITE-BlobRepositoryCloudTest-seed#[161D53EED566A556]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:44043 44043
   [junit4]   2> 756413 INFO  (Thread-709) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:44043 44043
   [junit4]   2> 756413 WARN  (Thread-709) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	2	/solr/aliases.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	3	/solr/collections/.system/state.json
   [junit4]   2> 	2	/solr/clusterstate.json
   [junit4]   2> 	2	/solr/clusterprops.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	2	/solr/live_nodes
   [junit4]   2> 	2	/solr/collections
   [junit4]   2> 
   [junit4]   2> 756413 INFO  (SUITE-BlobRepositoryCloudTest-seed#[161D53EED566A556]-worker) [    ] o.a.s.SolrTestCaseJ4 ------------------------------------------------------- Done waiting for tracked resources to be released
   [junit4]   2> Nov 18, 2017 2:32:44 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 2 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=212, maxMBSortInHeap=7.335572478505424, sim=RandomSimilarity(queryNorm=false): {}, locale=ar-LB, timezone=Asia/Manila
   [junit4]   2> NOTE: Linux 4.10.0-37-generic i386/Oracle Corporation 1.8.0_144 (32-bit)/cpus=8,threads=1,free=100297008,total=283746304
   [junit4]   2> NOTE: All tests run in this JVM: [TestLuceneIndexBackCompat, TestSchemaNameResource, PeerSyncTest, BlockDirectoryTest, SoftAutoCommitTest, TestComplexPhraseLeadingWildcard, MissingSegmentRecoveryTest, RAMDirectoryFactoryTest, HighlighterMaxOffsetTest, TestBadConfig, IgnoreCommitOptimizeUpdateProcessorFactoryTest, DistributedQueryComponentOptimizationTest, ConnectionManagerTest, TestQueryUtils, DistributedQueryElevationComponentTest, DataDrivenBlockJoinTest, CloudMLTQParserTest, TestStressCloudBlindAtomicUpdates, DistributedDebugComponentTest, TestSolrCloudWithHadoopAuthPlugin, ZkControllerTest, SpellCheckComponentTest, MinimalSchemaTest, CdcrBidirectionalTest, TestDownShardTolerantSearch, TestDFRSimilarityFactory, TestClusterProperties, TestTestInjection, TestSuggestSpellingConverter, MetricsHandlerTest, URLClassifyProcessorTest, BasicFunctionalityTest, TestRandomFaceting, BigEndianAscendingWordDeserializerTest, InfixSuggestersTest, IndexSchemaRuntimeFieldTest, TestLegacyField, TestFieldSortValues, ChaosMonkeyNothingIsSafeWithPullReplicasTest, TestPHPSerializedResponseWriter, BasicZkTest, TestIBSimilarityFactory, PrimitiveFieldTypeTest, TestOrdValues, SolrCoreTest, NodeLostTriggerTest, CleanupOldIndexTest, TestCustomStream, SearchHandlerTest, SaslZkACLProviderTest, BlobRepositoryCloudTest]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=BlobRepositoryCloudTest -Dtests.seed=161D53EED566A556 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=ar-LB -Dtests.timezone=Asia/Manila -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   0.00s J0 | BlobRepositoryCloudTest (suite) <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:35749/solr: create the collection time out:180s
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([161D53EED566A556]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1104)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:883)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:816)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
   [junit4]    > 	at org.apache.solr.core.BlobRepositoryCloudTest.setupCluster(BlobRepositoryCloudTest.java:51)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4] Completed [209/749 (1!)] on J0 in 211.20s, 0 tests, 1 error <<< FAILURES!

[...truncated 774 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderFailureAfterFreshStartTest
   [junit4]   2> Creating dataDir: /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailureAfterFreshStartTest_161D53EED566A556-001/init-core-data-001
   [junit4]   2> 1688446 WARN  (SUITE-LeaderFailureAfterFreshStartTest-seed#[161D53EED566A556]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=9 numCloses=9
   [junit4]   2> 1688446 INFO  (SUITE-LeaderFailureAfterFreshStartTest-seed#[161D53EED566A556]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 1688448 INFO  (SUITE-LeaderFailureAfterFreshStartTest-seed#[161D53EED566A556]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 1688448 INFO  (SUITE-LeaderFailureAfterFreshStartTest-seed#[161D53EED566A556]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 1688448 INFO  (SUITE-LeaderFailureAfterFreshStartTest-seed#[161D53EED566A556]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /sb_s/
   [junit4]   2> 1688454 INFO  (TEST-LeaderFailureAfterFreshStartTest.test-seed#[161D53EED566A556]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1688468 INFO  (Thread-1974) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1688468 INFO  (Thread-1974) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1688516 ERROR (Thread-1974) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 1688568 INFO  (TEST-LeaderFailureAfterFreshStartTest.test-seed#[161D53EED566A556]) [    ] o.a.s.c.ZkTestServer start zk server on port:35967
   [junit4]   2> 1688628 INFO  (TEST-LeaderFailureAfterFreshStartTest.test-seed#[161D53EED566A556]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1688629 INFO  (TEST-LeaderFailureAfterFreshStartTest.test-seed#[161D53EED566A556]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1688630 INFO  (TEST-LeaderFailureAfterFreshStartTest.test-seed#[161D53EED566A556]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1688631 INFO  (TEST-LeaderFailureAfterFreshStartTest.test-seed#[161D53EED566A556]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1688631 INFO  (TEST-LeaderFailureAfterFreshStartTest.test-seed#[161D53EED566A556]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1688632 INFO  (TEST-LeaderFailureAfterFreshStartTest.test-seed#[161D53EED566A556]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1688633 INFO  (TEST-LeaderFailureAfterFreshStartTest.test-seed#[161D53EED566A556]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1688633 INFO  (TEST-LeaderFailureAfterFreshStartTest.test-seed#[161D53EED566A556]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1688634 INFO  (TEST-LeaderFailureAfterFreshStartTest.test-seed#[161D53EED566A556]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1688634 INFO  (TEST-LeaderFailureAfterFreshStartTest.test-seed#[161D53EED566A556]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1688635 INFO  (TEST-LeaderFailureAfterFreshStartTest.test-seed#[161D53EED566A556]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1688671 INFO  (TEST-LeaderFailureAfterFreshStartTest.test-seed#[161D53EED566A556]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
   [junit4]   2> 1688777 INFO  (TEST-LeaderFailureAfterFreshStartTest.test-seed#[161D53EED566A556]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1688857 INFO  (TEST-LeaderFailureAfterFreshStartTest.test-seed#[161D53EED566A556]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5da20d{/sb_s,null,AVAILABLE}
   [junit4]   2> 1688862 INFO  (TEST-LeaderFailureAfterFreshStartTest.test-seed#[161D53EED566A556]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@4d77aa{SSL,[ssl, http/1.1]}{127.0.0.1:34365}
   [junit4]   2> 1688862 INFO  (TEST-LeaderFailureAfterFreshStartTest.test-seed#[161D53EED566A556]) [    ] o.e.j.s.Server Started @1690378ms
   [junit4]   2> 1688862 INFO  (TEST-LeaderFailureAfterFreshStartTest.test-seed#[161D53EED566A556]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailureAfterFreshStartTest_161D53EED566A556-001/tempDir-001/control/data, replicaType=NRT, hostContext=/sb_s, hostPort=34365, coreRootDirectory=/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/../../../../../../../../../home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailureAfterFreshStartTest_161D53EED566A556-001/control-001/cores}
   [junit4]   2> 1688863 ERROR (TEST-LeaderFailureAfterFreshStartTest.test-seed#[161D53EED566A556]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1688863 INFO  (TEST-LeaderFailureAfterFreshStartTest.test-seed#[161D53EED566A556]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.2.0
   [junit4]   2> 1688863 INFO  (TEST-LeaderFailureAfterFreshStartTest.test-seed#[161D53EED566A556]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1688863 INFO  (TEST-LeaderFailureAfterFreshStartTest.test-seed#[161D53EED566A556]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1688863 INFO  (TEST-LeaderFailureAfterFreshStartTest.test-seed#[161D53EED566A556]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-11-18T02:48:17.548Z
   [junit4]   2> 1688868 INFO  (TEST-LeaderFailureAfterFreshStartTest.test-seed#[161D53EED566A556]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 1688868 INFO  (TEST-LeaderFailureAfterFreshStartTest.test-seed#[161D53EED566A556]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailureAfterFreshStartTest_161D53EED566A556-001/control-001/solr.xml
   [junit4]   2> 1688872 INFO  (TEST-LeaderFailureAfterFreshStartTest.test-seed#[161D53EED566A556]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 1688872 INFO  (TEST-LeaderFailureAfterFreshStartTest.test-seed#[161D53EED566A556]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 1688888 INFO  (TEST-LeaderFailureAfterFreshStartTest.test-seed#[161D53EED566A556]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@14c06b6, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 1688891 INFO  (TEST-LeaderFailureAfterFreshStartTest.test-seed#[161D53EED566A556]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:35967/solr
   [junit4]   2> 1688931 INFO  (TEST-LeaderFailureAfterFreshStartTest.test-seed#[161D53EED566A556]) [n:127.0.0.1:34365_sb_s    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1688931 INFO  (TEST-LeaderFailureAfterFreshStartTest.test-seed#[161D53EED566A556]) [n:127.0.0.1:34365_sb_s    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:34365_sb_s
   [junit4]   2> 1688932 INFO  (TEST-LeaderFailureAfterFreshStartTest.test-seed#[161D53EED566A556]) [n:127.0.0.1:34365_sb_s    ] o.a.s.c.Overseer Overseer (id=99023146002481157-127.0.0.1:34365_sb_s-n_0000000000) starting
   [junit4]   2> 1688936 INFO  (TEST-LeaderFailureAfterFreshStartTest.test-seed#[161D53EED566A556]) [n:127.0.0.1:34365_sb_s    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:34365_sb_s
   [junit4]   2> 1688953 INFO  (zkCallback-1645-thread-1-processing-n:127.0.0.1:34365_sb_s) [n:127.0.0.1:34365_sb_s    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1689023 INFO  (TEST-LeaderFailureAfterFreshStartTest.test-seed#[161D53EED566A556]) [n:127.0.0.1:34365_sb_s    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14c06b6
   [junit4]   2> 1689031 INFO  (TEST-LeaderFailureAfterFreshStartTest.test-seed#[161D53EED566A556]) [n:127.0.0.1:34365_sb_s    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14c06b6
   [junit4]   2> 1689031 INFO  (TEST-LeaderFailureAfterFreshStartTest.test-seed#[161D53EED566A556]) [n:127.0.0.1:34365_sb_s    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14c06b6
   [junit4]   2> 1689032 INFO  (TEST-LeaderFailureAfterFreshStartTest.test-seed#[161D53EED566A556]) [n:127.0.0.1:34365_sb_s    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/../../../../../../../../../home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailureAfterFreshStartTest_161D53EED566A556-001/control-001/cores
   [junit4]   2> 1689046 INFO  (TEST-LeaderFailureAfterFreshStartTest.test-seed#[161D53EED566A556]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1689047 INFO  (TEST-LeaderFailureAfterFreshStartTest.test-seed#[161D53EED566A556]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:35967/solr ready
   [junit4]   2> 1689087 INFO  (qtp28101713-10739) [n:127.0.0.1:34365_sb_s    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:34365_sb_s&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1689088 INFO  (OverseerThreadFactory-3996-thread-1-processing-n:127.0.0.1:34365_sb_s) [n:127.0.0.1:34365_sb_s    ] o.a.s.c.CreateCollectionCmd Create collection control_collection
   [junit4]   2> 1689194 INFO  (qtp28101713-10738) [n:127.0.0.1:34365_sb_s    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1689195 INFO  (qtp28101713-10738) [n:127.0.0.1:34365_sb_s    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 1689299 INFO  (zkCallback-1645-thread-1-processing-n:127.0.0.1:34365_sb_s) [n:127.0.0.1:34365_sb_s    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1690205 INFO  (qtp28101713-10738) [n:127.0.0.1:34365_sb_s c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
   [junit4]   2> 1690214 INFO  (qtp28101713-10738) [n:127.0.0.1:34365_sb_s c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
   [junit4]   2> 1690302 INFO  (qtp28101713-10738) [n:127.0.0.1:34365_sb_s c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 1690309 INFO  (qtp28101713-10738) [n:127.0.0.1:34365_sb_s c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
   [junit4]   2> 1690310 INFO  (qtp28101713-10738) [n:127.0.0.1:34365_sb_s c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14c06b6
   [junit4]   2> 1690310 INFO  (qtp28101713-10738) [n:127.0.0.1:34365_sb_s c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1690310 INFO  (qtp28101713-10738) [n:127.0.0.1:34365_sb_s c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailureAfterFreshStartTest_161D53EED566A556-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/../../../../../../../../../home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailureAfterFreshStartTest_161D53EED566A556-001/control-001/cores/control_collection_shard1_replica_n1/data/]
   [junit4]   2> 1690313 INFO  (qtp28101713-10738) [n:127.0.0.1:34365_sb_s c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=11, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8826056013751066]
   [junit4]   2> 1690327 WARN  (qtp28101713-10738) [n:127.0.0.1:34365_sb_s c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 1690367 INFO  (qtp28101713-10738) [n:127.0.0.1:34365_sb_s c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1690367 INFO  (qtp28101713-10738) [n:127.0.0.1:34365_sb_s c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1690368 INFO  (qtp28101713-10738) [n:127.0.0.1:34365_sb_s c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1690368 INFO  (qtp28101713-10738) [n:127.0.0.1:34365_sb_s c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1690370 INFO  (qtp28101713-10738) [n:127.0.0.1:34365_sb_s c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=22, maxMergeAtOnceExplicit=31, maxMergedSegmentMB=57.171875, floorSegmentMB=1.6455078125, forceMergeDeletesPctAllowed=28.870361155477767, segmentsPerTier=10.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 1690370 INFO  (qtp28101713-10738) [n:127.0.0.1:34365_sb_s c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@1f8b0cf[control_collection_shard1_replica_n1] main]
   [junit4]   2> 1690371 INFO  (qtp28101713-10738) [n:127.0.0.1:34365_sb_s c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1690371 INFO  (qtp28101713-10738) [n:127.0.0.1:34365_sb_s c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]

[...truncated too long message...]

_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=630194
   [junit4]   2> 3401331 ERROR (TEST-TestCloudManagedSchema.test-seed#[161D53EED566A556]) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 3401331 INFO  (TEST-TestCloudManagedSchema.test-seed#[161D53EED566A556]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:37147 37147
   [junit4]   2> 3406455 INFO  (Thread-2190) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:37147 37147
   [junit4]   2> 3406456 WARN  (Thread-2190) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	2	/solr/aliases.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	2	/solr/clusterstate.json
   [junit4]   2> 	2	/solr/clusterprops.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	2	/solr/live_nodes
   [junit4]   2> 	2	/solr/collections
   [junit4]   2> 
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestCloudManagedSchema -Dtests.method=test -Dtests.seed=161D53EED566A556 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=ar-OM -Dtests.timezone=America/Argentina/Buenos_Aires -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR    636s J0 | TestCloudManagedSchema.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:36553/ozsr/e
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([161D53EED566A556:9E496C347B9AC8AE]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:654)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1104)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:883)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:816)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:314)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:991)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > Caused by: java.net.SocketTimeoutException: Read timed out
   [junit4]    > 	at java.net.SocketInputStream.socketRead0(Native Method)
   [junit4]    > 	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
   [junit4]    > 	at java.net.SocketInputStream.read(SocketInputStream.java:171)
   [junit4]    > 	at java.net.SocketInputStream.read(SocketInputStream.java:141)
   [junit4]    > 	at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
   [junit4]    > 	at sun.security.ssl.InputRecord.read(InputRecord.java:503)
   [junit4]    > 	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:983)
   [junit4]    > 	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:940)
   [junit4]    > 	at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
   [junit4]    > 	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
   [junit4]    > 	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
   [junit4]    > 	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
   [junit4]    > 	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
   [junit4]    > 	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
   [junit4]    > 	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
   [junit4]    > 	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
   [junit4]    > 	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
   [junit4]    > 	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
   [junit4]    > 	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:542)
   [junit4]    > 	... 43 more
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.schema.TestCloudManagedSchema_161D53EED566A556-001
   [junit4]   2> NOTE: test params are: codec=HighCompressionCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=HIGH_COMPRESSION, chunkSize=5, maxDocsPerChunk=451, blockSize=1), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=HIGH_COMPRESSION, chunkSize=5, blockSize=1)), sim=RandomSimilarity(queryNorm=false): {}, locale=ar-OM, timezone=America/Argentina/Buenos_Aires
   [junit4]   2> NOTE: Linux 4.10.0-37-generic i386/Oracle Corporation 1.8.0_144 (32-bit)/cpus=8,threads=1,free=65456904,total=330661888
   [junit4]   2> NOTE: All tests run in this JVM: [TestLuceneIndexBackCompat, TestSchemaNameResource, PeerSyncTest, BlockDirectoryTest, SoftAutoCommitTest, TestComplexPhraseLeadingWildcard, MissingSegmentRecoveryTest, RAMDirectoryFactoryTest, HighlighterMaxOffsetTest, TestBadConfig, IgnoreCommitOptimizeUpdateProcessorFactoryTest, DistributedQueryComponentOptimizationTest, ConnectionManagerTest, TestQueryUtils, DistributedQueryElevationComponentTest, DataDrivenBlockJoinTest, CloudMLTQParserTest, TestStressCloudBlindAtomicUpdates, DistributedDebugComponentTest, TestSolrCloudWithHadoopAuthPlugin, ZkControllerTest, SpellCheckComponentTest, MinimalSchemaTest, CdcrBidirectionalTest, TestDownShardTolerantSearch, TestDFRSimilarityFactory, TestClusterProperties, TestTestInjection, TestSuggestSpellingConverter, MetricsHandlerTest, URLClassifyProcessorTest, BasicFunctionalityTest, TestRandomFaceting, BigEndianAscendingWordDeserializerTest, InfixSuggestersTest, IndexSchemaRuntimeFieldTest, TestLegacyField, TestFieldSortValues, ChaosMonkeyNothingIsSafeWithPullReplicasTest, TestPHPSerializedResponseWriter, BasicZkTest, TestIBSimilarityFactory, PrimitiveFieldTypeTest, TestOrdValues, SolrCoreTest, NodeLostTriggerTest, CleanupOldIndexTest, TestCustomStream, SearchHandlerTest, SaslZkACLProviderTest, BlobRepositoryCloudTest, TestConfigSetProperties, AssignBackwardCompatibilityTest, AutoScalingHandlerTest, CoreAdminRequestStatusTest, TestSchemaVersionResource, ResponseHeaderTest, TestSolrCoreSnapshots, SmileWriterTest, TestPhraseSuggestions, TestRecovery, ActionThrottleTest, TestSolrConfigHandlerConcurrent, TestSolrJ, TestHashQParserPlugin, HdfsUnloadDistributedZkTest, TestMaxScoreQueryParser, RemoteQueryErrorTest, TestReloadAndDeleteDocs, TestWordDelimiterFilterFactory, AtomicUpdatesTest, SolrPluginUtilsTest, TestFieldCacheVsDocValues, SimpleCollectionCreateDeleteTest, LeaderFailoverAfterPartitionTest, HdfsRecoverLeaseTest, DistributedFacetPivotLargeTest, TestExportWriter, JvmMetricsTest, ExternalFileFieldSortTest, SolrShardReporterTest, TestCollectionAPIs, TestIndexingPerformance, TestRebalanceLeaders, EnumFieldTest, BasicDistributedZkTest, TestLeaderElectionWithEmptyReplica, TestEmbeddedSolrServerSchemaAPI, DocValuesTest, DeleteReplicaTest, TestDFISimilarityFactory, DistribCursorPagingTest, QueryParsingTest, SolrXmlInZkTest, TestBinaryField, TestSort, ResponseLogComponentTest, TestRequestStatusCollectionAPI, TestNoOpRegenerator, TestReloadDeadlock, HdfsAutoAddReplicasIntegrationTest, TestDistributedStatsComponentCardinality, JsonLoaderTest, TestSolrCloudWithDelegationTokens, MultiTermTest, SpellCheckCollatorTest, CollectionsAPISolrJTest, ZkFailoverTest, StatelessScriptUpdateProcessorFactoryTest, TestCloudManagedSchema]
   [junit4] Completed [662/749 (3!)] on J0 in 635.83s, 1 test, 1 error <<< FAILURES!

[...truncated 43410 lines...]

[JENKINS] Lucene-Solr-7.x-Linux (64bit/jdk-9.0.1) - Build # 850 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Linux/850/
Java: 64bit/jdk-9.0.1 -XX:-UseCompressedOops -XX:+UseG1GC

1 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.analytics.facet.RangeFacetTest

Error Message:
There are still nodes recoverying - waited for 90 seconds

Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 90 seconds
	at __randomizedtesting.SeedInfo.seed([53A8356709A986C4]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:185)
	at org.apache.solr.analytics.SolrAnalyticsTestCase.setupCollection(SolrAnalyticsTestCase.java:63)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:874)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.base/java.lang.Thread.run(Thread.java:844)




Build Log:
[...truncated 17140 lines...]
   [junit4] Suite: org.apache.solr.analytics.facet.RangeFacetTest
   [junit4]   2> Creating dataDir: /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/contrib/solr-analytics/test/J2/temp/solr.analytics.facet.RangeFacetTest_53A8356709A986C4-001/init-core-data-001
   [junit4]   1> / (12)
   [junit4]   1>  /configs (1)
   [junit4]   1>   /configs/conf (2)
   [junit4]   1>    /configs/conf/schema.xml (0)
   [junit4]   1>    DATA: ...supressed...
   [junit4]   1>    /configs/conf/solrconfig.xml (0)
   [junit4]   1>    DATA: ...supressed...
   [junit4]   1>  /overseer (6)
   [junit4]   1>  DATA:
   [junit4]   1>      
   [junit4]   1>   /overseer/collection-queue-work (0)
   [junit4]   1>   /overseer/queue-work (0)
   [junit4]   1>   /overseer/collection-map-failure (0)
   [junit4]   1>   /overseer/collection-map-completed (0)
   [junit4]   1>   /overseer/queue (2)
   [junit4]   1>    /overseer/queue/qn-0000000009 (0)
   [junit4]   1>    DATA:
   [junit4]   1>        {
   [junit4]   1>          "core":"collection1_shard1_replica_n1",
   [junit4]   1>          "core_node_name":"core_node2",
   [junit4]   1>          "roles":null,
   [junit4]   1>          "base_url":"https://127.0.0.1:34085/solr",
   [junit4]   1>          "node_name":"127.0.0.1:34085_solr",
   [junit4]   1>          "numShards":"2",
   [junit4]   1>          "state":"active",
   [junit4]   1>          "shard":"shard1",
   [junit4]   1>          "collection":"collection1",
   [junit4]   1>          "type":"NRT",
   [junit4]   1>          "operation":"state"}
   [junit4]   1>    /overseer/queue/qn-0000000008 (0)
   [junit4]   1>    DATA:
   [junit4]   1>        {
   [junit4]   1>          "core":"collection1_shard2_replica_n3",
   [junit4]   1>          "core_node_name":"core_node4",
   [junit4]   1>          "roles":null,
   [junit4]   1>          "base_url":"https://127.0.0.1:41133/solr",
   [junit4]   1>          "node_name":"127.0.0.1:41133_solr",
   [junit4]   1>          "numShards":"2",
   [junit4]   1>          "state":"active",
   [junit4]   1>          "shard":"shard2",
   [junit4]   1>          "collection":"collection1",
   [junit4]   1>          "type":"NRT",
   [junit4]   1>          "operation":"state"}
   [junit4]   1>   /overseer/collection-map-running (0)
   [junit4]   1>  /aliases.json (0)
   [junit4]   1>  /live_nodes (4)
   [junit4]   1>   /live_nodes/127.0.0.1:42161_solr (0)
   [junit4]   1>   /live_nodes/127.0.0.1:41133_solr (0)
   [junit4]   1>   /live_nodes/127.0.0.1:35497_solr (0)
   [junit4]   1>   /live_nodes/127.0.0.1:34085_solr (0)
   [junit4]   1>  /collections (1)
   [junit4]   1>   /collections/collection1 (4)
   [junit4]   1>   DATA:
   [junit4]   1>       {"configName":"conf"}
   [junit4]   1>    /collections/collection1/leaders (2)
   [junit4]   1>     /collections/collection1/leaders/shard2 (1)
   [junit4]   1>      /collections/collection1/leaders/shard2/leader (0)
   [junit4]   1>      DATA:
   [junit4]   1>          {
   [junit4]   1>            "core":"collection1_shard2_replica_n3",
   [junit4]   1>            "core_node_name":"core_node4",
   [junit4]   1>            "base_url":"https://127.0.0.1:41133/solr",
   [junit4]   1>            "node_name":"127.0.0.1:41133_solr"}
   [junit4]   1>     /collections/collection1/leaders/shard1 (1)
   [junit4]   1>      /collections/collection1/leaders/shard1/leader (0)
   [junit4]   1>      DATA:
   [junit4]   1>          {
   [junit4]   1>            "core":"collection1_shard1_replica_n1",
   [junit4]   1>            "core_node_name":"core_node2",
   [junit4]   1>            "base_url":"https://127.0.0.1:34085/solr",
   [junit4]   1>            "node_name":"127.0.0.1:34085_solr"}
   [junit4]   1>    /collections/collection1/counter (0)
   [junit4]   1>    DATA:
   [junit4]   1>        
   [junit4]   1>    /collections/collection1/state.json (0)
   [junit4]   1>    DATA:
   [junit4]   1>        {"collection1":{
   [junit4]   1>            "pullReplicas":"0",
   [junit4]   1>            "replicationFactor":"1",
   [junit4]   1>            "router":{"name":"compositeId"},
   [junit4]   1>            "maxShardsPerNode":"1",
   [junit4]   1>            "autoAddReplicas":"false",
   [junit4]   1>            "nrtReplicas":"1",
   [junit4]   1>            "tlogReplicas":"0",
   [junit4]   1>            "shards":{
   [junit4]   1>              "shard1":{
   [junit4]   1>                "range":"80000000-ffffffff",
   [junit4]   1>                "state":"active",
   [junit4]   1>                "replicas":{"core_node2":{
   [junit4]   1>                    "core":"collection1_shard1_replica_n1",
   [junit4]   1>                    "base_url":"https://127.0.0.1:34085/solr",
   [junit4]   1>                    "node_name":"127.0.0.1:34085_solr",
   [junit4]   1>                    "state":"down",
   [junit4]   1>                    "type":"NRT",
   [junit4]   1>                    "leader":"true"}}},
   [junit4]   1>              "shard2":{
   [junit4]   1>                "range":"0-7fffffff",
   [junit4]   1>                "state":"active",
   [junit4]   1>                "replicas":{"core_node4":{
   [junit4]   1>                    "core":"collection1_shard2_replica_n3",
   [junit4]   1>                    "base_url":"https://127.0.0.1:41133/solr",
   [junit4]   1>                    "node_name":"127.0.0.1:41133_solr",
   [junit4]   1>                    "state":"down",
   [junit4]   1>                    "type":"NRT",
   [junit4]   1>                    "leader":"true"}}}}}}
   [junit4]   1>    /collections/collection1/leader_elect (2)
   [junit4]   1>     /collections/collection1/leader_elect/shard2 (1)
   [junit4]   1>      /collections/collection1/leader_elect/shard2/election (1)
   [junit4]   1>       /collections/collection1/leader_elect/shard2/election/99027492440375305-core_node4-n_0000000000 (0)
   [junit4]   1>     /collections/collection1/leader_elect/shard1 (1)
   [junit4]   1>      /collections/collection1/leader_elect/shard1/election (1)
   [junit4]   1>       /collections/collection1/leader_elect/shard1/election/99027492440375307-core_node2-n_0000000000 (0)
   [junit4]   1>  /overseer_elect (2)
   [junit4]   1>   /overseer_elect/leader (0)
   [junit4]   1>   DATA:
   [junit4]   1>       {"id":"99027492440375305-127.0.0.1:41133_solr-n_0000000000"}
   [junit4]   1>   /overseer_elect/election (4)
   [junit4]   1>    /overseer_elect/election/99027492440375305-127.0.0.1:41133_solr-n_0000000000 (0)
   [junit4]   1>    /overseer_elect/election/99027492440375306-127.0.0.1:35497_solr-n_0000000001 (0)
   [junit4]   1>    /overseer_elect/election/99027492440375308-127.0.0.1:42161_solr-n_0000000002 (0)
   [junit4]   1>    /overseer_elect/election/99027492440375307-127.0.0.1:34085_solr-n_0000000003 (0)
   [junit4]   1>  /security.json (0)
   [junit4]   1>  DATA:
   [junit4]   1>      {}
   [junit4]   1>  /clusterstate.json (0)
   [junit4]   1>  DATA:
   [junit4]   1>      {}
   [junit4]   1>  /solr.xml (0)
   [junit4]   1>  DATA: ...supressed...
   [junit4]   1>  /autoscaling (4)
   [junit4]   1>   /autoscaling/nodeAdded (0)
   [junit4]   1>   /autoscaling/nodeLost (0)
   [junit4]   1>   /autoscaling/triggerState (1)
   [junit4]   1>    /autoscaling/triggerState/.auto_add_replicas (0)
   [junit4]   1>    DATA:
   [junit4]   1>        {
   [junit4]   1>          "lastLiveNodes":[
   [junit4]   1>            "127.0.0.1:34085_solr",
   [junit4]   1>            "127.0.0.1:35497_solr",
   [junit4]   1>            "127.0.0.1:41133_solr",
   [junit4]   1>            "127.0.0.1:42161_solr"],
   [junit4]   1>          "nodeNameVsTimeRemoved":{}}
   [junit4]   1>   /autoscaling/events (1)
   [junit4]   1>    /autoscaling/events/.auto_add_replicas (0)
   [junit4]   1>  /autoscaling.json (0)
   [junit4]   1>  DATA:
   [junit4]   1>      {
   [junit4]   1>        "cluster-preferences":[{
   [junit4]   1>            "minimize":"cores",
   [junit4]   1>            "precision":1}],
   [junit4]   1>        "triggers":{".auto_add_replicas":{
   [junit4]   1>            "name":".auto_add_replicas",
   [junit4]   1>            "event":"nodeLost",
   [junit4]   1>            "waitFor":30,
   [junit4]   1>            "actions":[
   [junit4]   1>              {
   [junit4]   1>                "name":"auto_add_replicas_plan",
   [junit4]   1>                "class":"solr.AutoAddReplicasPlanAction"},
   [junit4]   1>              {
   [junit4]   1>                "name":"execute_plan",
   [junit4]   1>                "class":"solr.ExecutePlanAction"}],
   [junit4]   1>            "enabled":true}},
   [junit4]   1>        "listeners":{".auto_add_replicas.system":{
   [junit4]   1>            "beforeAction":[],
   [junit4]   1>            "afterAction":[],
   [junit4]   1>            "trigger":".auto_add_replicas",
   [junit4]   1>            "stage":[
   [junit4]   1>              "STARTED",
   [junit4]   1>              "ABORTED",
   [junit4]   1>              "SUCCEEDED",
   [junit4]   1>              "FAILED",
   [junit4]   1>              "BEFORE_ACTION",
   [junit4]   1>              "AFTER_ACTION",
   [junit4]   1>              "IGNORED"],
   [junit4]   1>            "class":"org.apache.solr.cloud.autoscaling.SystemLogListener"}}}
   [junit4]   1>  /clusterprops.json (0)
   [junit4]   1>  DATA:
   [junit4]   1>      {'urlScheme':'https'}
   [junit4]   1> 
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=560, maxMBSortInHeap=6.743603304200372, sim=RandomSimilarity(queryNorm=false): {}, locale=rn, timezone=Singapore
   [junit4]   2> NOTE: Linux 4.10.0-37-generic amd64/Oracle Corporation 9.0.1 (64-bit)/cpus=8,threads=1,free=247757576,total=536870912
   [junit4]   2> NOTE: All tests run in this JVM: [CastingIntValueStreamTest, BooleanFieldsTest, ConstantValueTest, GTEFunctionTest, LegacyFacetSortingTest, FloatFieldsTest, TopFunctionTest, CastingLongValueTest, RangeFacetTest]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RangeFacetTest -Dtests.seed=53A8356709A986C4 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=rn -Dtests.timezone=Singapore -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   0.00s J2 | RangeFacetTest (suite) <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: There are still nodes recoverying - waited for 90 seconds
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([53A8356709A986C4]:0)
   [junit4]    > 	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:185)
   [junit4]    > 	at org.apache.solr.analytics.SolrAnalyticsTestCase.setupCollection(SolrAnalyticsTestCase.java:63)
   [junit4]    > 	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]    > 	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]    > 	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]    > 	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
   [junit4]    > 	at java.base/java.lang.Thread.run(Thread.java:844)
   [junit4] Completed [74/78 (1!)] on J2 in 127.74s, 0 tests, 1 failure <<< FAILURES!

[...truncated 37087 lines...]

[JENKINS] Lucene-Solr-7.x-Linux (64bit/jdk1.8.0_144) - Build # 849 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Linux/849/
Java: 64bit/jdk1.8.0_144 -XX:-UseCompressedOops -XX:+UseG1GC

5 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.core.TestLazyCores

Error Message:
1 thread leaked from SUITE scope at org.apache.solr.core.TestLazyCores:     1) Thread[id=12209, name=searcherExecutor-4305-thread-1, state=WAITING, group=TGRP-TestLazyCores]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.core.TestLazyCores: 
   1) Thread[id=12209, name=searcherExecutor-4305-thread-1, state=WAITING, group=TGRP-TestLazyCores]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([3F315409024E6EEE]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.core.TestLazyCores

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=12209, name=searcherExecutor-4305-thread-1, state=WAITING, group=TGRP-TestLazyCores]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=12209, name=searcherExecutor-4305-thread-1, state=WAITING, group=TGRP-TestLazyCores]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([3F315409024E6EEE]:0)


FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI

Error Message:
Error from server at https://127.0.0.1:34115/solr/awhollynewcollection_0_shard2_replica_n2: ClusterState says we are the leader (https://127.0.0.1:34115/solr/awhollynewcollection_0_shard2_replica_n2), but locally we don't think so. Request came from null

Stack Trace:
org.apache.solr.client.solrj.impl.CloudSolrClient$RouteException: Error from server at https://127.0.0.1:34115/solr/awhollynewcollection_0_shard2_replica_n2: ClusterState says we are the leader (https://127.0.0.1:34115/solr/awhollynewcollection_0_shard2_replica_n2), but locally we don't think so. Request came from null
	at __randomizedtesting.SeedInfo.seed([3F315409024E6EEE:774420BD047D417B]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:549)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1012)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:883)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:945)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:945)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:945)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:945)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:945)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:816)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.request.UpdateRequest.commit(UpdateRequest.java:233)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI(CollectionsAPIDistributedZkTest.java:459)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:34115/solr/awhollynewcollection_0_shard2_replica_n2: ClusterState says we are the leader (https://127.0.0.1:34115/solr/awhollynewcollection_0_shard2_replica_n2), but locally we don't think so. Request came from null
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.lambda$directUpdate$0(CloudSolrClient.java:526)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	... 1 more


FAILED:  org.apache.solr.cloud.MoveReplicaHDFSFailoverTest.testDataDirAndUlogAreMaintained

Error Message:
Error from server at https://127.0.0.1:41283/solr: Could not fully create collection: movereplicatest_coll2

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:41283/solr: Could not fully create collection: movereplicatest_coll2
	at __randomizedtesting.SeedInfo.seed([3F315409024E6EEE:1002C1DABE3EDCCC]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1103)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:883)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:816)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
	at org.apache.solr.cloud.MoveReplicaHDFSFailoverTest.testDataDirAndUlogAreMaintained(MoveReplicaHDFSFailoverTest.java:75)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.core.TestLazyCores.testNoCommit

Error Message:
Exception during query

Stack Trace:
java.lang.RuntimeException: Exception during query
	at __randomizedtesting.SeedInfo.seed([3F315409024E6EEE:E051F5D8C9690D4B]:0)
	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:901)
	at org.apache.solr.core.TestLazyCores.check10(TestLazyCores.java:847)
	at org.apache.solr.core.TestLazyCores.testNoCommit(TestLazyCores.java:829)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.RuntimeException: REQUEST FAILED: xpath=//result[@numFound='10']
	xml response was: <?xml version="1.0" encoding="UTF-8"?>
<response>
<lst name="responseHeader"><int name="status">0</int><int name="QTime">0</int><lst name="params"><str name="q">*:*</str></lst></lst><result name="response" numFound="0" start="0"></result>
</response>

	request was:q=*:*
	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:894)
	... 41 more




Build Log:
[...truncated 12204 lines...]
   [junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest
   [junit4]   2> Creating dataDir: /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_3F315409024E6EEE-001/init-core-data-001
   [junit4]   2> 535186 WARN  (SUITE-CollectionsAPIDistributedZkTest-seed#[3F315409024E6EEE]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
   [junit4]   2> 535186 INFO  (SUITE-CollectionsAPIDistributedZkTest-seed#[3F315409024E6EEE]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 535187 INFO  (SUITE-CollectionsAPIDistributedZkTest-seed#[3F315409024E6EEE]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 535188 INFO  (SUITE-CollectionsAPIDistributedZkTest-seed#[3F315409024E6EEE]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 535189 INFO  (SUITE-CollectionsAPIDistributedZkTest-seed#[3F315409024E6EEE]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 4 servers in /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_3F315409024E6EEE-001/tempDir-001
   [junit4]   2> 535189 INFO  (SUITE-CollectionsAPIDistributedZkTest-seed#[3F315409024E6EEE]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 535190 INFO  (Thread-1829) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 535190 INFO  (Thread-1829) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 535195 ERROR (Thread-1829) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 535290 INFO  (SUITE-CollectionsAPIDistributedZkTest-seed#[3F315409024E6EEE]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:45893
   [junit4]   2> 535298 INFO  (jetty-launcher-582-thread-2) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 535298 INFO  (jetty-launcher-582-thread-3) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 535302 INFO  (jetty-launcher-582-thread-4) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 535303 INFO  (jetty-launcher-582-thread-1) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 535313 INFO  (jetty-launcher-582-thread-3) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3bbdb8ce{/solr,null,AVAILABLE}
   [junit4]   2> 535314 INFO  (jetty-launcher-582-thread-3) [    ] o.e.j.s.AbstractConnector Started ServerConnector@2d604908{SSL,[ssl, http/1.1]}{127.0.0.1:38413}
   [junit4]   2> 535314 INFO  (jetty-launcher-582-thread-3) [    ] o.e.j.s.Server Started @536874ms
   [junit4]   2> 535314 INFO  (jetty-launcher-582-thread-3) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=38413}
   [junit4]   2> 535315 ERROR (jetty-launcher-582-thread-3) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 535315 INFO  (jetty-launcher-582-thread-3) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solrâ„¢ version 7.2.0
   [junit4]   2> 535315 INFO  (jetty-launcher-582-thread-3) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 535315 INFO  (jetty-launcher-582-thread-3) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 535315 INFO  (jetty-launcher-582-thread-3) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-11-18T16:56:31.248Z
   [junit4]   2> 535321 INFO  (jetty-launcher-582-thread-4) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@ebd97ed{/solr,null,AVAILABLE}
   [junit4]   2> 535321 INFO  (jetty-launcher-582-thread-4) [    ] o.e.j.s.AbstractConnector Started ServerConnector@4986fff6{SSL,[ssl, http/1.1]}{127.0.0.1:34115}
   [junit4]   2> 535321 INFO  (jetty-launcher-582-thread-4) [    ] o.e.j.s.Server Started @536881ms
   [junit4]   2> 535321 INFO  (jetty-launcher-582-thread-4) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=34115}
   [junit4]   2> 535322 ERROR (jetty-launcher-582-thread-4) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 535322 INFO  (jetty-launcher-582-thread-4) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solrâ„¢ version 7.2.0
   [junit4]   2> 535322 INFO  (jetty-launcher-582-thread-4) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 535322 INFO  (jetty-launcher-582-thread-4) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 535322 INFO  (jetty-launcher-582-thread-4) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-11-18T16:56:31.255Z
   [junit4]   2> 535328 INFO  (jetty-launcher-582-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@77193a35{/solr,null,AVAILABLE}
   [junit4]   2> 535329 INFO  (jetty-launcher-582-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5d25d8ae{/solr,null,AVAILABLE}
   [junit4]   2> 535329 INFO  (jetty-launcher-582-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@33f8910{SSL,[ssl, http/1.1]}{127.0.0.1:37371}
   [junit4]   2> 535329 INFO  (jetty-launcher-582-thread-1) [    ] o.e.j.s.Server Started @536888ms
   [junit4]   2> 535329 INFO  (jetty-launcher-582-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=37371}
   [junit4]   2> 535329 INFO  (jetty-launcher-582-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@121d35ad{SSL,[ssl, http/1.1]}{127.0.0.1:39609}
   [junit4]   2> 535329 INFO  (jetty-launcher-582-thread-2) [    ] o.e.j.s.Server Started @536888ms
   [junit4]   2> 535329 INFO  (jetty-launcher-582-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=39609}
   [junit4]   2> 535329 ERROR (jetty-launcher-582-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 535329 INFO  (jetty-launcher-582-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solrâ„¢ version 7.2.0
   [junit4]   2> 535329 INFO  (jetty-launcher-582-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 535329 INFO  (jetty-launcher-582-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 535329 ERROR (jetty-launcher-582-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 535329 INFO  (jetty-launcher-582-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-11-18T16:56:31.262Z
   [junit4]   2> 535329 INFO  (jetty-launcher-582-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solrâ„¢ version 7.2.0
   [junit4]   2> 535329 INFO  (jetty-launcher-582-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 535329 INFO  (jetty-launcher-582-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 535329 INFO  (jetty-launcher-582-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-11-18T16:56:31.262Z
   [junit4]   2> 535331 INFO  (jetty-launcher-582-thread-4) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 535333 INFO  (jetty-launcher-582-thread-4) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 535333 INFO  (jetty-launcher-582-thread-4) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 535337 INFO  (jetty-launcher-582-thread-3) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 535340 INFO  (jetty-launcher-582-thread-3) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 535340 INFO  (jetty-launcher-582-thread-3) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 535360 INFO  (jetty-launcher-582-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 535360 INFO  (jetty-launcher-582-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 535362 INFO  (jetty-launcher-582-thread-4) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45893/solr
   [junit4]   2> 535363 INFO  (jetty-launcher-582-thread-3) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45893/solr
   [junit4]   2> 535366 INFO  (jetty-launcher-582-thread-2) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 535366 INFO  (jetty-launcher-582-thread-1) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 535366 INFO  (jetty-launcher-582-thread-2) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 535366 INFO  (jetty-launcher-582-thread-1) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 535374 INFO  (jetty-launcher-582-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45893/solr
   [junit4]   2> 535379 INFO  (jetty-launcher-582-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45893/solr
   [junit4]   2> 535386 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15fd00f66940008, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 535394 INFO  (jetty-launcher-582-thread-3) [n:127.0.0.1:38413_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 535395 INFO  (jetty-launcher-582-thread-3) [n:127.0.0.1:38413_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:38413_solr
   [junit4]   2> 535396 INFO  (jetty-launcher-582-thread-3) [n:127.0.0.1:38413_solr    ] o.a.s.c.Overseer Overseer (id=99026481389502473-127.0.0.1:38413_solr-n_0000000000) starting
   [junit4]   2> 535398 INFO  (jetty-launcher-582-thread-2) [n:127.0.0.1:39609_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 535399 INFO  (jetty-launcher-582-thread-1) [n:127.0.0.1:37371_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 535399 INFO  (jetty-launcher-582-thread-2) [n:127.0.0.1:39609_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:39609_solr
   [junit4]   2> 535399 INFO  (jetty-launcher-582-thread-1) [n:127.0.0.1:37371_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37371_solr
   [junit4]   2> 535402 INFO  (zkCallback-600-thread-1-processing-n:127.0.0.1:38413_solr) [n:127.0.0.1:38413_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 535404 INFO  (jetty-launcher-582-thread-3) [n:127.0.0.1:38413_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38413_solr
   [junit4]   2> 535405 INFO  (zkCallback-600-thread-1-processing-n:127.0.0.1:38413_solr) [n:127.0.0.1:38413_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 535430 INFO  (zkCallback-606-thread-1-processing-n:127.0.0.1:39609_solr) [n:127.0.0.1:39609_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 535430 INFO  (zkCallback-604-thread-1-processing-n:127.0.0.1:37371_solr) [n:127.0.0.1:37371_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 535448 INFO  (zkCallback-604-thread-2-processing-n:127.0.0.1:37371_solr) [n:127.0.0.1:37371_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 535449 INFO  (zkCallback-606-thread-2-processing-n:127.0.0.1:39609_solr) [n:127.0.0.1:39609_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 535454 INFO  (jetty-launcher-582-thread-4) [n:127.0.0.1:34115_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 535454 INFO  (jetty-launcher-582-thread-4) [n:127.0.0.1:34115_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 535455 INFO  (jetty-launcher-582-thread-4) [n:127.0.0.1:34115_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:34115_solr
   [junit4]   2> 535473 INFO  (zkCallback-595-thread-1-processing-n:127.0.0.1:34115_solr) [n:127.0.0.1:34115_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 535473 INFO  (zkCallback-600-thread-2-processing-n:127.0.0.1:38413_solr) [n:127.0.0.1:38413_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 535473 INFO  (zkCallback-606-thread-2-processing-n:127.0.0.1:39609_solr) [n:127.0.0.1:39609_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 535473 INFO  (zkCallback-604-thread-2-processing-n:127.0.0.1:37371_solr) [n:127.0.0.1:37371_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 535489 WARN  (jetty-launcher-582-thread-3) [n:127.0.0.1:38413_solr    ] o.a.s.m.r.j.JmxMetricsReporter Unable to register timer
   [junit4]   2> javax.management.InstanceNotFoundException: solr:dom1=node,category=UPDATE,scope=updateShardHandler,name=threadPool.recoveryExecutor.duration
   [junit4]   2> 	at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getMBean(DefaultMBeanServerInterceptor.java:1095)
   [junit4]   2> 	at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.exclusiveUnregisterMBean(DefaultMBeanServerInterceptor.java:427)
   [junit4]   2> 	at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.unregisterMBean(DefaultMBeanServerInterceptor.java:415)
   [junit4]   2> 	at com.sun.jmx.mbeanserver.JmxMBeanServer.unregisterMBean(JmxMBeanServer.java:546)
   [junit4]   2> 	at org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.registerMBean(JmxMetricsReporter.java:530)
   [junit4]   2> 	at org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.onTimerAdded(JmxMetricsReporter.java:670)
   [junit4]   2> 	at com.codahale.metrics.MetricRegistry.notifyListenerOfAddedMetric(MetricRegistry.java:462)
   [junit4]   2> 	at com.codahale.metrics.MetricRegistry.addListener(MetricRegistry.java:297)
   [junit4]   2> 	at org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter.start(JmxMetricsReporter.java:725)
   [junit4]   2> 	at org.apache.solr.metrics.reporters.SolrJmxReporter.doInit(SolrJmxReporter.java:109)
   [junit4]   2> 	at org.apache.solr.metrics.SolrMetricReporter.init(SolrMetricReporter.java:70)
   [junit4]   2> 	at org.apache.solr.metrics.SolrMetricManager.loadReporter(SolrMetricManager.java:883)
   [junit4]   2> 	at org.apache.solr.metrics.SolrMetricManager.loadReporters(SolrMetricManager.java:819)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:548)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:263)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:183)
   [junit4]   2> 	at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1596)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1659)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1316)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1145)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)
   [junit4]   2> 	at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)
   [junit4]   2> 	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)
   [junit4]   2> 	at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:384)
   [junit4]   2> 	at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:247)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 535492 INFO  (jetty-launcher-582-thread-2) [n:127.0.0.1:39609_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3881d057
   [junit4]   2> 535513 INFO  (jetty-launcher-582-thread-3) [n:127.0.0.1:38413_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3881d057
   [junit4]   2> 535514 INFO  (jetty-launcher-582-thread-2) [n:127.0.0.1:39609_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3881d057
   [junit4]   2> 535514 INFO  (jetty-launcher-582-thread-2) [n:127.0.0.1:39609_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3881d057
   [junit4]   2> 535515 INFO  (jetty-launcher-582-thread-2) [n:127.0.0.1:39609_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_3F315409024E6EEE-001/tempDir-001/node1
   [junit4]   2> 535522 INFO  (jetty-launcher-582-thread-3) [n:127.0.0.1:38413_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3881d057
   [junit4]   2> 535522 INFO  (jetty-launcher-582-thread-3) [n:127.0.0.1:38413_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3881d057
   [junit4]   2> 535523 INFO  (jetty-launcher-582-thread-3) [n:127.0.0.1:38413_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_3F315409024E6EEE-001/tempDir-001/node3
   [junit4]   2> 535552 INFO  (jetty-launcher-582-thread-1) [n:127.0.0.1:37371_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3881d057
   [junit4]   2> 535554 INFO  (jetty-launcher-582-thread-4) [n:127.0.0.1:34115_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3881d057
   [junit4]   2> 535560 INFO  (jetty-launcher-582-thread-1) [n:127.0.0.1:37371_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3881d057
   [junit4]   2> 535560 INFO  (jetty-launcher-582-thread-1) [n:127.0.0.1:37371_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3881d057
   [junit4]   2> 535561 INFO  (jetty-launcher-582-thread-1) [n:127.0.0.1:37371_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_3F315409024E6EEE-001/tempDir-001/node2
   [junit4]   2> 535563 INFO  (jetty-launcher-582-thread-4) [n:127.0.0.1:34115_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3881d057
   [junit4]   2> 535563 INFO  (jetty-launcher-582-thread-4) [n:127.0.0.1:34115_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3881d057
   [junit4]   2> 535564 INFO  (jetty-launcher-582-thread-4) [n:127.0.0.1:34115_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_3F315409024E6EEE-001/tempDir-001/node4
   [junit4]   2> 535662 INFO  (SUITE-CollectionsAPIDistributedZkTest-seed#[3F315409024E6EEE]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 535662 INFO  (SUITE-CollectionsAPIDistributedZkTest-seed#[3F315409024E6EEE]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45893/solr ready
   [junit4]   2> 535714 INFO  (TEST-CollectionsAPIDistributedZkTest.testBadActionNames-seed#[3F315409024E6EEE]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testBadActionNames
   [junit4]   2> 535717 INFO  (TEST-CollectionsAPIDistributedZkTest.testBadActionNames-seed#[3F315409024E6EEE]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 535746 ERROR (qtp1943445707-5296) [n:127.0.0.1:37371_solr    ] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: Unknown action: BADACTION
   [junit4]   2> 	at org.apache.solr.handler.admin.CollectionsHandler.handleRequestBody(CollectionsHandler.java:217)
   [junit4]   2> 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:735)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:716)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:497)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:493)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 
   [junit4]   2> 535747 INFO  (qtp1943445707-5296) [n:127.0.0.1:37371_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={name=badactioncollection&action=BADACTION&numShards=2&wt=javabin&version=2} status=400 QTime=0
   [junit4]   2> 535747 INFO  (TEST-CollectionsAPIDistributedZkTest.testBadActionNames-seed#[3F315409024E6EEE]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testBadActionNames
   [junit4]   2> 535783 INFO  (TEST-CollectionsAPIDistributedZkTest.addReplicaTest-seed#[3F315409024E6EEE]) [    ] o.a.s.SolrTestCaseJ4 ###Starting addReplicaTest
   [junit4]   2> 535784 INFO  (TEST-CollectionsAPIDistributedZkTest.addReplicaTest-seed#[3F315409024E6EEE]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 535785 INFO  (qtp1943445707-5311) [n:127.0.0.1:37371_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=2&collection.configName=conf&maxShardsPerNode=4&name=addReplicaColl&nrtReplicas=2&action=CREATE&numShards=2&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 535787 INFO  (OverseerThreadFactory-1625-thread-1-processing-n:127.0.0.1:38413_solr) [n:127.0.0.1:38413_solr    ] o.a.s.c.CreateCollectionCmd Create collection addReplicaColl
   [junit4]   2> 535892 INFO  (OverseerStateUpdate-99026481389502473-127.0.0.1:38413_solr-n_0000000000) [n:127.0.0.1:38413_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"addReplicaColl",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"addReplicaColl_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:39609/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 535893 INFO  (OverseerStateUpdate-99026481389502473-127.0.0.1:38413_solr-n_0000000000) [n:127.0.0.1:38413_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"addReplicaColl",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"addReplicaColl_shard1_replica_n2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:37371/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 535895 INFO  (OverseerStateUpdate-99026481389502473-127.0.0.1:38413_solr-n_0000000000) [n:127.0.0.1:38413_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"addReplicaColl",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"addReplicaColl_shard2_replica_n4",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:38413/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 535901 INFO  (OverseerStateUpdate-99026481389502473-127.0.0.1:38413_solr-n_0000000000) [n:127.0.0.1:38413_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"addReplicaColl",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"addReplicaColl_shard2_replica_n6",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:34115/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 536111 INFO  (qtp1943445707-5296) [n:127.0.0.1:37371_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node5&collection.configName=conf&newCollection=true&name=addReplicaColl_shard1_replica_n2&action=CREATE&numShards=2&collection=addReplicaColl&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 536111 INFO  (qtp1943445707-5296) [n:127.0.0.1:37371_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 536137 INFO  (qtp2088294172-5301) [n:127.0.0.1:38413_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node7&collection.configName=conf&newCollection=true&name=addReplicaColl_shard2_replica_n4&action=CREATE&numShards=2&collection=addReplicaColl&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 536137 INFO  (qtp2088294172-5301) [n:127.0.0.1:38413_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 536139 INFO  (qtp621262729-5293) [n:127.0.0.1:34115_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node8&collection.configName=conf&newCollection=true&name=addReplicaColl_shard2_replica_n6&action=CREATE&numShards=2&collection=addReplicaColl&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 536140 INFO  (qtp621262729-5293) [n:127.0.0.1:34115_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 536145 INFO  (qtp538674982-5312) [n:127.0.0.1:39609_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=addReplicaColl_shard1_replica_n1&action=CREATE&numShards=2&collection=addReplicaColl&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 536146 INFO  (qtp538674982-5312) [n:127.0.0.1:39609_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 536247 INFO  (zkCallback-604-thread-2-processing-n:127.0.0.1:37371_solr) [n:127.0.0.1:37371_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/addReplicaColl/state.json] for collection [addReplicaColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 536247 INFO  (zkCallback-606-thread-2-processing-n:127.0.0.1:39609_solr) [n:127.0.0.1:39609_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/addReplicaColl/state.json] for collection [addReplicaColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 536248 INFO  (zkCallback-595-thread-1-processing-n:127.0.0.1:34115_solr) [n:127.0.0.1:34115_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/addReplicaColl/state.json] for collection [addReplicaColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 536248 INFO  (zkCallback-606-thread-1-processing-n:127.0.0.1:39609_solr) [n:127.0.0.1:39609_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/addReplicaColl/state.json] for collection [addReplicaColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 536248 INFO  (zkCallback-595-thread-2-processing-n:127.0.0.1:34115_solr) [n:127.0.0.1:34115_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/addReplicaColl/state.json] for collection [addReplicaColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 536247 INFO  (zkCallback-604-thread-1-processing-n:127.0.0.1:37371_solr) [n:127.0.0.1:37371_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/addReplicaColl/state.json] for collection [addReplicaColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 536248 INFO  (zkCallback-600-thread-2-processing-n:127.0.0.1:38413_solr) [n:127.0.0.1:38413_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/addReplicaColl/state.json] for collection [addReplicaColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 536248 INFO  (zkCallback-600-thread-1-processing-n:127.0.0.1:38413_solr) [n:127.0.0.1:38413_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/addReplicaColl/state.json] for collection [addReplicaColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 537122 INFO  (qtp1943445707-5296) [n:127.0.0.1:37371_solr c:addReplicaColl s:shard1 r:core_node5 x:addReplicaColl_shard1_replica_n2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
   [junit4]   2> 537127 INFO  (qtp1943445707-5296) [n:127.0.0.1:37371_solr c:addReplicaColl s:shard1 r:core_node5 x:addReplicaColl_shard1_replica_n2] o.a.s.s.IndexSchema [addReplicaColl_shard1_replica_n2] Schema name=minimal
   [junit4]   2> 537128 INFO  (qtp1943445707-5296) [n:127.0.0.1:37371_solr c:addReplicaColl s:shard1 r:core_node5 x:addReplicaColl_shard1_replica_n2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 537128 INFO  (qtp1943445707-5296) [n:127.0.0.1:37371_solr c:addReplicaColl s:shard1 r:core_node5 x:addReplicaColl_shard1_replica_n2] o.a.s.c.CoreContainer Creating SolrCore 'addReplicaColl_shard1_replica_n2' using configuration from collection addReplicaColl, trusted=true
   [junit4]   2> 537129 INFO  (qtp1943445707-5296) [n:127.0.0.1:37371_solr c:addReplicaColl s:shard1 r:core_node5 x:addReplicaColl_shard1_replica_n2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.addReplicaColl.shard1.replica_n2' (registry 'solr.core.addReplicaColl.shard1.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3881d057
   [junit4]   2> 537129 INFO  (qtp1943445707-5296) [n:127.0.0.1:37371_solr c:addReplicaColl s:shard1 r:core_node5 x:addReplicaColl_shard1_replica_n2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 537129 INFO  (qtp1943445707-5296) [n:127.0.0.1:37371_solr c:addReplicaColl s:shard1 r:core_node5 x:addReplicaColl_shard1_replica_n2] o.a.s.c.SolrCore [[addReplicaColl_shard1_replica_n2] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_3F315409024E6EEE-001/tempDir-001/node2/addReplicaColl_shard1_replica_n2], dataDir=[/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_3F315409024E6EEE-001/tempDir-001/node2/addReplicaColl_shard1_replica_n2/data/]
   [junit4]   2> 537155 INFO  (qtp621262729-5293) [n:127.0.0.1:34115_solr c:addReplicaColl s:shard2 r:core_node8 x:addReplicaColl_shard2_replica_n6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
   [junit4]   2> 537158 INFO  (qtp2088294172-5301) [n:127.0.0.1:38413_solr c:addReplicaColl s:shard2 r:core_node7 x:addReplicaColl_shard2_replica_n4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
   [junit4]   2> 537163 INFO  (qtp538674982-5312) [n:127.0.0.1:39609_solr c:addReplicaColl s:shard1 r:core_node3 x:addReplicaColl_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
   [junit4]   2> 537163 INFO  (qtp621262729-5293) [n:127.0.0.1:34115_solr c:addReplicaColl s:shard2 r:core_node8 x:addReplicaColl_shard2_replica_n6] o.a.s.s.IndexSchema [addReplicaColl_shard2_replica_n6] Schema name=minimal
   [junit4]   2> 537166 INFO  (qtp621262729-5293) [n:127.0.0.1:34115_solr c:addReplicaColl s:shard2 r:core_node8 x:addReplicaColl_shard2_replica_n6] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 537166 INFO  (qtp621262729-5293) [n:127.0.0.1:34115_solr c:addReplicaColl s:shard2 r:core_node8 x:addReplicaColl_shard2_replica_n6] o.a.s.c.CoreContainer Creating SolrCore 'addReplicaColl_shard2_replica_n6' using configuration from collection addReplicaColl, trusted=true
   [junit4]   2> 537166 INFO  (qtp621262729-5293) [n:127.0.0.1:34115_solr c:addReplicaColl s:shard2 r:core_node8 x:addReplicaColl_shard2_replica_n6] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.addReplicaColl.shard2.replica_n6' (registry 'solr.core.addReplicaColl.shard2.replica_n6') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3881d057
   [junit4]   2> 537167 INFO  (qtp621262729-5293) [n:127.0.0.1:34115_solr c:addReplicaColl s:shard2 r:core_node8 x:addReplicaColl_shard2_replica_n6] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 537167 INFO  (qtp621262729-5293) [n:127.0.0.1:34115_solr c:addReplicaColl s:shard2 r:core_node8 x:addReplicaColl_shard2_replica_n6] o.a.s.c.SolrCore [[addReplicaColl_shard2_replica_n6] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_3F315409024E6EEE-001/tempDir-001/node4/addReplicaColl_shard2_replica_n6], dataDir=[/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_3F315409024E6EEE-001/tempDir-001/node4/addReplicaColl_shard2_replica_n6/data/]
   [junit4]   2> 537173 INFO  (qtp1943445707-5296) [n:127.0.0.1:37371_solr c:addReplicaColl s:shard1 r:core_node5 x:addReplicaColl_shard1_replica_n2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 537173 INFO  (qtp1943445707-5296) [n:127.0.0.1:37371_solr c:addReplicaColl s:shard1 r:core_node5 x:addReplicaColl_shard1_replica_n2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 537175 INFO  (qtp1943445707-5296) [n:127.0.0.1:37371_solr c:addReplicaColl s:shard1 r:core_node5 x:addReplicaColl_shard1_replica_n2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 537175 INFO  (qtp1943445707-5296) [n:127.0.0.1:37371_solr c:addReplicaColl s:shard1 r:core_node5 x:addReplicaColl_shard1_replica_n2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 537177 INFO  (qtp1943445707-5296) [n:127.0.0.1:37371_solr c:addReplicaColl s:shard1 r:core_node5 x:addReplicaColl_shard1_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@52c8b37d[addReplicaColl_shard1_replica_n2] main]
   [junit4]   2> 537177 INFO  (qtp1943445707-5296) [n:127.0.0.1:37371_solr c:addReplicaColl s:shard1 r:core_node5 x:addReplicaColl_shard1_replica_n2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 537177 INFO  (qtp1943445707-5296) [n:127.0.0.1:37371_solr c:addReplicaColl s:shard1 r:core_node5 x:addReplicaColl_shard1_replica_n2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 537178 INFO  (qtp1943445707-5296) [n:127.0.0.1:37371_solr c:addReplicaColl s:shard1 r:core_node5 x:addReplicaColl_shard1_replica_n2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 537179 INFO  (qtp2088294172-5301) [n:127.0.0.1:38413_solr c:addReplicaColl s:shard2 r:core_node7 x:addReplicaColl_shard2_replica_n4] o.a.s.s.IndexSchema [addReplicaColl_shard2_replica_n4] Schema name=minimal
   [junit4]   2> 537180 INFO  (qtp538674982-5312) [n:127.0.0.1:39609_solr c:addReplicaColl s:shard1 r:core_node3 x:addReplicaColl_shard1_replica_n1] o.a.s.s.IndexSchema [addReplicaColl_shard1_replica_n1] Schema name=minimal
   [junit4]   2> 537180 INFO  (searcherExecutor-1634-thread-1-processing-n:127.0.0.1:37371_solr x:addReplicaColl_shard1_replica_n2 s:shard1 c:addReplicaColl r:core_node5) [n:127.0.0.1:37371_solr c:addReplicaColl s:shard1 r:core_node5 x:addReplicaColl_shard1_replica_n2] o.a.s.c.SolrCore [addReplicaColl_shard1_replica_n2] Registered new searcher Searcher@52c8b37d[addReplicaColl_shard1_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 537180 INFO  (qtp1943445707-5296) [n:127.0.0.1:37371_solr c:addReplicaColl s:shard1 r:core_node5 x:addReplicaColl_shard1_replica_n2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1584423704317657088
   [junit4]   2> 537218 INFO  (qtp2088294172-5301) [n:127.0.0.1:38413_solr c:addReplicaColl s:shard2 r:core_node7 x:addReplicaColl_shard2_replica_n4] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 537218 INFO  (qtp538674982-5312) [n:127.0.0.1:39609_solr c:addReplicaColl s:shard1 r:core_node3 x:addReplicaColl_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 537218 INFO  (qtp2088294172-5301) [n:127.0.0.1:38413_solr c:addReplicaColl s:shard2 r:core_node7 x:addReplicaColl_shard2_replica_n4] o.a.s.c.CoreContainer Creating SolrCore 'addReplicaColl_shard2_replica_n4' using configuration from collection addReplicaColl, trusted=true
   [junit4]   2> 537218 INFO  (qtp538674982-5312) [n:127.0.0.1:39609_solr c:addReplicaColl s:shard1 r:core_node3 x:addReplicaColl_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'addReplicaColl_shard1_replica_n1' using configuration from collection addReplicaColl, trusted=true
   [junit4]   2> 537218 INFO  (qtp538674982-5312) [n:127.0.0.1:39609_solr c:addReplicaColl s:shard1 r:core_node3 x:addReplicaColl_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.addReplicaColl.shard1.replica_n1' (registry 'solr.core.addReplicaColl.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3881d057
   [junit4]   2> 537218 INFO  (qtp538674982-5312) [n:127.0.0.1:39609_solr c:addReplicaColl s:shard1 r:core_node3 x:addReplicaColl_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 537218 INFO  (qtp538674982-5312) [n:127.0.0.1:39609_solr c:addReplicaColl s:shard1 r:core_node3 x:addReplicaColl_shard1_replica_n1] o.a.s.c.SolrCore [[addReplicaColl_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_3F315409024E6EEE-001/tempDir-001/node1/addReplicaColl_shard1_replica_n1], dataDir=[/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_3F315409024E6EEE-001/tempDir-001/node1/addReplicaColl_shard1_replica_n1/data/]
   [junit4]   2> 537219 INFO  (qtp2088294172-5301) [n:127.0.0.1:38413_solr c:addReplicaColl s:shard2 r:core_node7 x:addReplicaColl_shard2_replica_n4] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.addReplicaColl.shard2.replica_n4' (registry 'solr.core.addReplicaColl.shard2.replica_n4') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3881d057
   [junit4]   2> 537219 INFO  (qtp2088294172-5301) [n:127.0.0.1:38413_solr c:addReplicaColl s:shard2 r:core_node7 x:addReplicaColl_shard2_replica_n4] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 537219 INFO  (qtp2088294172-5301) [n:127.0.0.1:38413_solr c:addReplicaColl s:shard2 r:core_node7 x:addReplicaColl_shard2_replica_n4] o.a.s.c.SolrCore [[addReplicaColl_shard2_replica_n4] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_3F315409024E6EEE-001/tempDir-001/node3/addReplicaColl_shard2_replica_n4], dataDir=[/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_3F315409024E6EEE-001/tempDir-001/node3/addReplicaColl_shard2_replica_n4/data/]
   [junit4]   2> 537219 INFO  (qtp1943445707-5296) [n:127.0.0.1:37371_solr c:addReplicaColl s:shard1 r:core_node5 x:addReplicaColl_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard1: total=2 found=1 timeoutin=9999ms
   [junit4]   2> 537251 INFO  (qtp621262729-5293) [n:127.0.0.1:34115_solr c:addReplicaColl s:shard2 r:core_node8 x:addReplicaColl_shard2_replica_n6] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 537251 INFO  (qtp621262729-5293) [n:127.0.0.1:34115_solr c:addReplicaColl s:shard2 r:core_node8 x:addReplicaColl_shard2_replica_n6] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 537252 INFO  (qtp2088294172-5301) [n:127.0.0.1:38413_solr c:addReplicaColl s:shard2 r:core_node7 x:addReplicaColl_shard2_replica_n4] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 537252 INFO  (qtp2088294172-5301) [n:127.0.0.1:38413_solr c:addReplicaColl s:shard2 r:core_node7 x:addReplicaColl_shard2_replica_n4] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 537253 INFO  (qtp621262729-5293) [n:127.0.0.1:34115_solr c:addReplicaColl s:shard2 r:core_node8 x:addReplicaColl_shard2_replica_n6] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 537253 INFO  (qtp621262729-5293) [n:127.0.0.1:34115_solr c:addReplicaColl s:shard2 r:core_node8 x:addReplicaColl_shard2_replica_n6] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 537254 INFO  (qtp2088294172-5301) [n:127.0.0.1:38413_solr c:addReplicaColl s:shard2 r:core_node7 x:addReplicaColl_shard2_replica_n4] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 537254 INFO  (qtp2088294172-5301) [n:127.0.0.1:38413_solr c:addReplicaColl s:shard2 r:core_node7 x:addReplicaColl_shard2_replica_n4] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 537256 INFO  (qtp2088294172-5301) [n:127.0.0.1:38413_solr c:addReplicaColl s:shard2 r:core_node7 x:addReplicaColl_shard2_replica_n4] o.a.s.s.SolrIndexSearcher Opening [Searcher@c32c905[addReplicaColl_shard2_replica_n4] main]
   [junit4]   2> 537257 INFO  (qtp621262729-5293) [n:127.0.0.1:34115_solr c:addReplicaColl s:shard2 r:core_node8 x:addReplicaColl_shard2_replica_n6] o.a.s.s.SolrIndexSearcher Opening [Searcher@d910702[addReplicaColl_shard2_replica_n6] main]
   [junit4]   2> 537257 INFO  (qtp2088294172-5301) [n:127.0.0.1:38413_solr c:addReplicaColl s:shard2 r:core_node7 x:addReplicaColl_shard2_replica_n4] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 537258 INFO  (qtp2088294172-5301) [n:127.0.0.1:38413_solr c:addReplicaColl s:shard2 r:core_node7 x:addReplicaColl_shard2_replica_n4] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 537259 INFO  (qtp2088294172-5301) [n:127.0.0.1:38413_solr c:addReplicaColl s:shard2 r:core_node7 x:addReplicaColl_shard2_replica_n4] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 537260 INFO  (qtp621262729-5293) [n:127.0.0.1:34115_solr c:addReplicaColl s:shard2 r:core_node8 x:addReplicaColl_shard2_replica_n6] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 537260 INFO  (searcherExecutor-1641-thread-1-processing-n:127.0.0.1:38413_solr x:addReplicaColl_shard2_replica_n4 s:shard2 c:addReplicaColl r:core_node7) [n:127.0.0.1:38413_solr c:addReplicaColl s:shard2 r:core_node7 x:addReplicaColl_shard2_replica_n4] o.a.s.c.SolrCore [addReplicaColl_shard2_replica_n4] Registered new searcher Searcher@c32c905[addReplicaColl_shard2_replica_n4] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 537260 INFO  (qtp621262729-5293) [n:127.0.0.1:34115_solr c:addReplicaColl s:shard2 r:core_node8 x:addReplicaColl_shard2_replica_n6] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 537261 INFO  (qtp621262729-5293) [n:127.0.0.1:34115_solr c:addReplicaColl s:shard2 r:core_node8 x:addReplicaColl_shard2_replica_n6] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 537262 INFO  (qtp2088294172-5301) [n:127.0.0.1:38413_solr c:addReplicaColl s:shard2 r:core_node7 x:addReplicaColl_shard2_replica_n4] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1584423704403640320
   [junit4]   2> 537263 INFO  (searcherExecutor-1636-thread-1-processing-n:127.0.0.1:34115_solr x:addReplicaColl_shard2_replica_n6 s:shard2 c:addReplicaColl r:core_node8) [n:127.0.0.1:34115_solr c:addReplicaColl s:shard2 r:core_node8 x:addReplicaColl_shard2_replica_n6] o.a.s.c.SolrCore [addReplicaColl_shard2_replica_n6] Registered new searcher Searcher@d910702[addReplicaColl_shard2_replica_n6] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 537264 INFO  (qtp621262729-5293) [n:127.0.0.1:34115_solr c:addReplicaColl s:shard2 r:core_node8 x:addReplicaColl_shard2_replica_n6] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1584423704405737472
   [junit4]   2> 537266 INFO  (qtp538674982-5312) [n:127.0.0.1:39609_solr c:addReplicaColl s:shard1 r:core_node3 x:addReplicaColl_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 537266 INFO  (qtp538674982-5312) [n:127.0.0.1:39609_solr c:addReplicaColl s:shard1 r:core_node3 x:addReplicaColl_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 537268 INFO  (qtp2088294172-5301) [n:127.0.0.1:38413_solr c:addReplicaColl s:shard2 r:core_node7 x:addReplicaColl_shard2_replica_n4] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 537268 INFO  (qtp2088294172-5301) [n:127.0.0.1:38413_solr c:addReplicaColl s:shard2 r:core_node7 x:addReplicaColl_shard2_replica_n4] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 537268 INFO  (qtp2088294172-5301) [n:127.0.0.1:38413_solr c:addReplicaColl s:shard2 r:core_node7 x:addReplicaColl_shard2_replica_n4] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:38413/solr/addReplicaColl_shard2_replica_n4/
   [junit4]   2> 537268 INFO  (qtp2088294172-5301) [n:127.0.0.1:38413_solr c:addReplicaColl s:shard2 r:core_node7 x:addReplicaColl_shard2_replica_n4] o.a.s.u.PeerSync PeerSync: core=addReplicaColl_shard2_replica_n4 url=https://127.0.0.1:38413/solr START replicas=[https://127.0.0.1:34115/solr/addReplicaColl_shard2_replica_n6/] nUpdates=100
   [junit4]   2> 537268 INFO  (qtp538674982-5312) [n:127.0.0.1:39609_solr c:addReplicaColl s:shard1 r:core_node3 x:addReplicaColl_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 537269 INFO  (qtp538674982-5312) [n:127.0.0.1:39609_solr c:addReplicaColl s:shard1 r:core_node3 x:addReplicaColl_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 537273 INFO  (qtp621262729-5300) [n:127.0.0.1:34115_solr c:addReplicaColl s:shard2 r:core_node8 x:addReplicaColl_shard2_replica_n6] o.a.s.c.S.Request [addReplicaColl_shard2_replica_n6]  webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=1
   [junit4]   2> 537273 INFO  (qtp538674982-5312) [n:127.0.0.1:39609_solr c:addReplicaColl s:shard1 r:core_node3 x:addReplicaColl_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@61e83470[addReplicaColl_shard1_replica_n1] main]
   [junit4]   2> 537274 INFO  (qtp538674982-5312) [n:127.0.0.1:39609_solr c:addReplicaColl s:shard1 r:core_node3 x:addReplicaColl_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 537274 INFO  (qtp538674982-5312) [n:127.0.0.1:39609_solr c:addReplicaColl s:shard1 r:core_node3 x:addReplicaColl_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 537275 INFO  (qtp538674982-5312) [n:127.0.0.1:39609_solr c:addReplicaColl s:shard1 r:core_node3 x:addReplicaColl_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 537277 INFO  (searcherExecutor-1640-thread-1-processing-n:127.0.0.1:39609_solr x:addReplicaColl_shard1_replica_n1 s:shard1 c:addReplicaColl r:core_node3) [n:127.0.0.1:39609_solr c:addReplicaColl s:shard1 r:core_node3 x:addReplicaColl_shard1_replica_n1] o.a.s.c.SolrCore [addReplicaColl_shard1_replica_n1] Registered new searcher Searcher@61e83470[addReplicaColl_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 537277 INFO  (qtp538674982-5312) [n:127.0.0.1:39609_solr c:addReplicaColl s:shard1 r:core_node3 x:addReplicaColl_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1584423704419368960
   [junit4]   2> 537369 INFO  (zkCallback-604-thread-1-processing-n:127.0.0.1:37371_solr) [n:127.0.0.1:37371_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/addReplicaColl/state.json] for collection [addReplicaColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 537369 INFO  (zkCallback-595-thread-2-processing-n:127.0.0.1:34115_solr) [n:127.0.0.1:34115_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/addReplicaColl/state.json] for collection [addReplicaColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 537369 INFO  (zkCallback-604-thread-2-processing-n:127.0.0.1:37371_solr) [n:127.0.0.1:37371_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/addReplicaColl/state.json] for collection [addReplicaColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 537369 INFO  (zkCallback-595-thread-1-processing-n:127.0.0.1:34115_solr) [n:127.0.0.1:34115_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/addReplicaColl/state.json] for collection [addReplicaColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 537369 INFO  (zkCallback-600-thread-1-processing-n:127.0.0.1:38413_solr) [n:127.0.0.1:38413_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/addReplicaColl/state.json] for collection [addReplicaColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 537369 INFO  (zkCallback-600-thread-2-processing-n:127.0.0.1:38413_solr) [n:127.0.0.1:38413_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/addReplicaColl/state.json] for collection [addReplicaColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 537369 INFO  (zkCallback-606-thread-2-processing-n:127.0.0.1:39609_solr) [n:127.0.0.1:39609_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/addReplicaColl/state.json] for collection [addReplicaColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 537369 INFO  (zkCallback-606-thread-1-processing-n:127.0.0.1:39609_solr) [n:127.0.0.1:39609_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/addReplicaColl/state.json] for collection [addReplicaColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 537568 INFO  (qtp2088294172-5301) [n:127.0.0.1:38413_solr c:addReplicaColl s:shard2 r:core_node7 x:addReplicaColl_shard2_replica_n4] o.a.s.u.PeerSync PeerSync: core=addReplicaColl_shard2_replica_n4 url=https://127.0.0.1:38413/solr DONE.  We have no versions.  sync failed.
   [junit4]   2> 537569 INFO  (qtp2088294172-5301) [n:127.0.0.1:38413_solr c:addReplicaColl s:shard2 r:core_node7 x:addReplicaColl_shard2_replica_n4] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
   [junit4]   2> 537569 INFO  (qtp2088294172-5301) [n:127.0.0.1:38413_solr c:addReplicaColl s:shard2 r:core_node7 x:addReplicaColl_shard2_replica_n4] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 537569 INFO  (qtp2088294172-5301) [n:127.0.0.1:38413_solr c:addReplicaColl s:shard2 r:core_node7 x:addReplicaColl_shard2_replica_n4] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 537571 INFO  (qtp2088294172-5301) [n:127.0.0.1:38413_solr c:addReplicaColl s:shard2 r:core_node7 x:addReplicaColl_shard2_replica_n4] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:38413/solr/addReplicaColl_shard2_replica_n4/ shard2
   [junit4]   2> 537672 INFO  (zkCallback-595-thread-2-processing-n:127.0.0.1:34115_solr) [n:127.0.0.1:34115_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/addReplicaColl/state.json] for collection [addReplicaColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 537672 INFO  (zkCallback-604-thread-2-processing-n:127.0.0.1:37371_solr) [n:127.0.0.1:37371_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/addReplicaColl/state.json] for collection [addReplicaColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 537672 INFO  (zkCallback-606-thread-2-processing-n:127.0.0.1:39609_solr) [n:127.0.0.1:39609_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/addReplicaColl/state.json] for collection [addReplicaColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 537672 INFO  (zkCallback-595-thread-1-processing-n:127.0.0.1:34115_solr) [n:127.0.0.1:34115_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/addReplicaColl/state.json] for collection [addReplicaColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 537672 INFO  (zkCallback-600-thread-2-processing-n:127.0.0.1:38413_solr) [n:127.0.0.1:38413_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/addReplicaColl/state.json] for collection [addReplicaColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 537672 INFO  (zkCallback-604-thread-1-processing-n:127.0.0.1:37371_solr) [n:127.0.0.1:37371_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/addReplicaColl/state.json] for collection [addReplicaColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 537672 INFO  (zkCallback-606-thread-1-processing-n:127.0.0.1:39609_solr) [n:127.0.0.1:39609_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/addReplicaColl/state.json] for collection [addReplicaColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 537672 INFO  (zkCallback-600-thread-3-processing-n:127.0.0.1:38413_solr) [n:127.0.0.1:38413_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/addReplicaColl/state.json] for collection [addReplicaColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 537720 INFO  (qtp1943445707-5296) [n:127.0.0.1:37371_solr c:addReplicaColl s:shard1 r:core_node5 x:addReplicaColl_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 537720 INFO  (qtp1943445707-5296) [n:127.0.0.1:37371_solr c:addReplicaColl s:shard1 r:core_node5 x:addReplicaColl_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 537720 INFO  (qtp1943445707-5296) [n:127.0.0.1:37371_solr c:addReplicaColl s:shard1 r:core_node5 x:addReplicaColl_shard1_replica_n2] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:37371/solr/addReplicaColl_shard1_replica_n2/
   [junit4]   2> 537720 INFO  (qtp1943445707-5296) [n:127.0.0.1:37371_solr c:addReplicaColl s:shard1 r:core_node5 x:addReplicaColl_shard1_replica_n2] o.a.s.u.PeerSync PeerSync: core=addReplicaColl_shard1_replica_n2 url=https://127.0.0.1:37371/solr START replicas=[https://127.0.0.1:39609/solr/addReplicaColl_shard1_replica_n1/] nUpdates=100
   [junit4]   2> 537722 INFO  (qtp2088294172-5301) [n:127.0.0.1:38413_solr c:addReplicaColl s:shard2 r:core_node7 x:addReplicaColl_shard2_replica_n4] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 537724 INFO  (qtp538674982-5309) [n:127.0.0.1:39609_solr c:addReplicaColl s:shard1 r:core_node3 x:addReplicaColl_shard1_replica_n1] o.a.s.c.S.Request [addReplicaColl_shard1_replica_n1]  webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 537793 INFO  (qtp2088294172-5301) [n:127.0.0.1:38413_solr c:addReplicaColl s:shard2 r:core_node7 x:addReplicaColl_shard2_replica_n4] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node7&collection.configName=conf&newCollection=true&name=addReplicaColl_shard2_replica_n4&action=CREATE&numShards=2&collection=addReplicaColl&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1656
   [junit4]   2> 537825 INFO  (zkCallback-606-thread-2-processing-n:127.0.0.1:39609_solr) [n:127.0.0.1:39609_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/addReplicaColl/state.json] for collection [addReplicaColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 537825 INFO  (zkCallback-604-thread-1-processing-n:127.0.0.1:37371_solr) [n:127.0.0.1:37371_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/addReplicaColl/state.json] for collection [addReplicaColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 537825 INFO  (zkCallback-606-thread-1-processing-n:127.0.0.1:39609_solr) [n:127.0.0.1:39609_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/addReplicaColl/state.json] for collection [addReplicaColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 537825 INFO  (zkCallback-600-thread-3-processing-n:127.0.0.1:38413_solr) [n:127.0.0.1:38413_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/addReplicaColl/state.json] for collection [addReplicaColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 537826 INFO  (zkCallback-604-thread-2-processing-n:127.0.0.1:37371_solr) [n:127.0.0.1:37371_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/addReplicaColl/state.json] for collection [addReplicaColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 537825 INFO  (zkCallback-595-thread-1-processing-n:127.0.0.1:34115_solr) [n:127.0.0.1:34115_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/addReplicaColl/state.json] for collection [addReplicaColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 537825 INFO  (zkCallback-600-thread-2-processing-n:127.0.0.1:38413_solr) [n:127.0.0.1:38413_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/addReplicaColl/state.json] for collection [addReplicaColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 537825 INFO  (zkCallback-595-thread-2-processing-n:127.0.0.1:34115_solr) [n:127.0.0.1:34115_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/addReplicaColl/state.json] for collection [addReplicaColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 538020 INFO  (qtp1943445707-5296) [n:127.0.0.1:37371_solr c:addReplicaColl s:shard1 r:core_node5 x:addReplicaColl_shard1_replica_n2] o.a.s.u.PeerSync PeerSync: core=addReplicaColl_shard1_replica_n2 url=https://127.0.0.1:37371/solr DONE.  We have no versions.  sync failed.
   [junit4]   2> 538020 INFO  (qtp1943445707-5296) [n:127.0.0.1:37371_solr c:addReplicaColl s:shard1 r:core_node5 x:addReplicaColl_shard1_replica_n2] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
   [junit4]   2> 538020 INFO  (qtp1943445707-5296) [n:127.0.0.1:37371_solr c:addReplicaColl s:shard1 r:core_node5 x:addReplicaColl_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 538020 INFO  (qtp1943445707-5296) [n:127.0.0.1:37371_solr c:addReplicaColl s:shard1 r:core_node5 x:addReplicaColl_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 538022 INFO  (qtp1943445707-5296) [n:127.0.0.1:37371_solr c:addReplicaColl s:shard1 r:core_node5 x:addReplicaColl_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:37371/solr/addReplicaColl_shard1_replica_n2/ shard1
   [junit4]   2> 538123 INFO  (zkCallback-595-thread-1-processing-n:127.0.0.1:34115_solr) [n:127.0.0.1:34115_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected ty

[...truncated too long message...]

836801 INFO  (TEST-TestLazyCores.testMidUseUnload-seed#[3F315409024E6EEE]) [    x:collection3] o.a.s.SolrTestCaseJ4 ###Ending testMidUseUnload
   [junit4]   2> 1836802 INFO  (SUITE-TestLazyCores-seed#[3F315409024E6EEE]-worker) [    x:collection3] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> 1836802 INFO  (SUITE-TestLazyCores-seed#[3F315409024E6EEE]-worker) [    x:collection3] o.a.s.c.CoreContainer Shutting down CoreContainer instance=555961484
   [junit4]   2> 1836802 INFO  (SUITE-TestLazyCores-seed#[3F315409024E6EEE]-worker) [    x:collection3] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 1836802 INFO  (SUITE-TestLazyCores-seed#[3F315409024E6EEE]-worker) [    x:collection3] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 1836802 INFO  (SUITE-TestLazyCores-seed#[3F315409024E6EEE]-worker) [    x:collection3] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_3F315409024E6EEE-001
   [junit4]   2> Nov 18, 2017 5:18:12 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> Nov 18, 2017 5:18:32 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 1 thread leaked from SUITE scope at org.apache.solr.core.TestLazyCores: 
   [junit4]   2>    1) Thread[id=12209, name=searcherExecutor-4305-thread-1, state=WAITING, group=TGRP-TestLazyCores]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]   2>         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Nov 18, 2017 5:18:32 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=12209, name=searcherExecutor-4305-thread-1, state=WAITING, group=TGRP-TestLazyCores]
   [junit4]   2> Nov 18, 2017 5:18:35 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> SEVERE: There are still zombie threads that couldn't be terminated:
   [junit4]   2>    1) Thread[id=12209, name=searcherExecutor-4305-thread-1, state=WAITING, group=TGRP-TestLazyCores]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]   2>         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: test params are: codec=DummyCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=DUMMY, chunkSize=32295, maxDocsPerChunk=958, blockSize=6), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=DUMMY, chunkSize=32295, blockSize=6)), sim=RandomSimilarity(queryNorm=true): {}, locale=nl-NL, timezone=Africa/Tunis
   [junit4]   2> NOTE: Linux 4.10.0-37-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=8,threads=2,free=144699648,total=536870912
   [junit4]   2> NOTE: All tests run in this JVM: [TestManagedSynonymFilterFactory, SolrRequestParserTest, OpenExchangeRatesOrgProviderTest, HttpTriggerListenerTest, ConjunctionSolrSpellCheckerTest, TestJsonFacets, SolrSlf4jReporterTest, TestSearcherReuse, AssignTest, TestInitQParser, TestSystemCollAutoCreate, TestDeleteCollectionOnDownNodes, HdfsDirectoryFactoryTest, OutputWriterTest, SuggestComponentContextFilterQueryTest, TestTolerantUpdateProcessorCloud, ClusterStateUpdateTest, TestSubQueryTransformer, DeleteShardTest, SampleTest, SolrIndexConfigTest, RegexBytesRefFilterTest, TestDynamicFieldResource, RegexBoostProcessorTest, ManagedSchemaRoundRobinCloudTest, TestLegacyFieldReuse, HdfsThreadLeakTest, TestBlendedInfixSuggestions, TestNumericTokenStream, SolrGraphiteReporterTest, OpenCloseCoreStressTest, BigEndianAscendingWordSerializerTest, TestCloudRecovery, DistributedExpandComponentTest, HdfsRecoveryZkTest, TestIntervalFaceting, TestXmlQParser, LegacyCloudClusterPropTest, TestSolr4Spatial2, PeerSyncReplicationTest, SpellingQueryConverterTest, SparseHLLTest, DirectUpdateHandlerTest, TestMissingGroups, PrimUtilsTest, CdcrRequestHandlerTest, SolrMetricReporterTest, PolyFieldTest, TestGeoJSONResponseWriter, HdfsBasicDistributedZkTest, TestAuthenticationFramework, CloudExitableDirectoryReaderTest, DOMUtilTest, ReplaceNodeTest, TestRTimerTree, TestFieldCacheWithThreads, TestBulkSchemaAPI, TestStandardQParsers, TestRetrieveFieldsOptimizer, TestSurroundQueryParser, TestDocSet, TestExpandComponent, TestFilteredDocIdSet, TestSerializedLuceneMatchVersion, CoreMergeIndexesAdminHandlerTest, CollectionsAPIDistributedZkTest, TestSimpleTrackingShardHandler, DocumentAnalysisRequestHandlerTest, AutoscalingHistoryHandlerTest, TestAnalyzedSuggestions, BooleanFieldTest, PathHierarchyTokenizerFactoryTest, TestCloudJSONFacetJoinDomain, TestMacros, CollectionTooManyReplicasTest, TriggerIntegrationTest, TestDocumentBuilder, WrapperMergePolicyFactoryTest, FullHLLTest, UtilsToolTest, TestReRankQParserPlugin, ConvertedLegacyTest, TestFieldTypeCollectionResource, PeerSyncWithIndexFingerprintCachingTest, BlockJoinFacetRandomTest, TestSQLHandlerNonCloud, TestPKIAuthenticationPlugin, ShardSplitTest, TestCloudSchemaless, TestRangeQuery, TestUseDocValuesAsStored, AutoAddReplicasIntegrationTest, AnalysisErrorHandlingTest, TestOmitPositions, CdcrBootstrapTest, MoveReplicaHDFSFailoverTest, SignatureUpdateProcessorFactoryTest, TestPseudoReturnFields, DistributedVersionInfoTest, TestOverriddenPrefixQueryForCustomFieldType, TestSSLRandomization, TestPushWriter, TestMultiValuedNumericRangeQuery, TestDelegationWithHadoopAuth, TestBM25SimilarityFactory, TestConfigSets, TestFastLRUCache, TestUnifiedSolrHighlighter, TestZkAclsWithHadoopAuth, TestAddFieldRealTimeGet, TestLeaderElectionZkExpiry, MetricsConfigTest, PropertiesRequestHandlerTest, TestAtomicUpdateErrorCases, TestLazyCores]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLazyCores -Dtests.seed=3F315409024E6EEE -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=nl-NL -Dtests.timezone=Africa/Tunis -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J1 | TestLazyCores (suite) <<<
   [junit4]    > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.core.TestLazyCores: 
   [junit4]    >    1) Thread[id=12209, name=searcherExecutor-4305-thread-1, state=WAITING, group=TGRP-TestLazyCores]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([3F315409024E6EEE]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   [junit4]    >    1) Thread[id=12209, name=searcherExecutor-4305-thread-1, state=WAITING, group=TGRP-TestLazyCores]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([3F315409024E6EEE]:0)
   [junit4] Completed [526/749 (3!)] on J1 in 33.39s, 9 tests, 3 errors <<< FAILURES!

[...truncated 48111 lines...]

[JENKINS] Lucene-Solr-7.x-Linux (64bit/jdk1.8.0_144) - Build # 848 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Linux/848/
Java: 64bit/jdk1.8.0_144 -XX:+UseCompressedOops -XX:+UseParallelGC

2 tests failed.
FAILED:  org.apache.solr.cloud.CollectionReloadTest.testReloadedLeaderStateAfterZkSessionLoss

Error Message:
Error from server at http://127.0.0.1:34713/solr: create the collection time out:180s

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:34713/solr: create the collection time out:180s
	at __randomizedtesting.SeedInfo.seed([30CBC7BAB2D589AA:CBE57CE71438FAFF]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1103)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:883)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:816)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
	at org.apache.solr.cloud.CollectionReloadTest.testReloadedLeaderStateAfterZkSessionLoss(CollectionReloadTest.java:53)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.update.processor.TimePartitionedUpdateProcessorTest.test

Error Message:
expected:<1> but was:<0>

Stack Trace:
java.lang.AssertionError: expected:<1> but was:<0>
	at __randomizedtesting.SeedInfo.seed([30CBC7BAB2D589AA:B89FF8601C29E452]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.update.processor.TimePartitionedUpdateProcessorTest.assertInvariants(TimePartitionedUpdateProcessorTest.java:245)
	at org.apache.solr.update.processor.TimePartitionedUpdateProcessorTest.test(TimePartitionedUpdateProcessorTest.java:123)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 11617 lines...]
   [junit4] Suite: org.apache.solr.cloud.CollectionReloadTest
   [junit4]   2> Creating dataDir: /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionReloadTest_30CBC7BAB2D589AA-001/init-core-data-001
   [junit4]   2> 6999 WARN  (SUITE-CollectionReloadTest-seed#[30CBC7BAB2D589AA]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
   [junit4]   2> 7000 INFO  (SUITE-CollectionReloadTest-seed#[30CBC7BAB2D589AA]-worker) [    ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 7001 INFO  (SUITE-CollectionReloadTest-seed#[30CBC7BAB2D589AA]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
   [junit4]   2> 7001 INFO  (SUITE-CollectionReloadTest-seed#[30CBC7BAB2D589AA]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 7005 INFO  (SUITE-CollectionReloadTest-seed#[30CBC7BAB2D589AA]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 1 servers in /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionReloadTest_30CBC7BAB2D589AA-001/tempDir-001
   [junit4]   2> 7027 INFO  (SUITE-CollectionReloadTest-seed#[30CBC7BAB2D589AA]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 7045 INFO  (Thread-12) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 7045 INFO  (Thread-12) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 7140 INFO  (SUITE-CollectionReloadTest-seed#[30CBC7BAB2D589AA]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:40041
   [junit4]   2> 7169 ERROR (Thread-12) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 7225 INFO  (jetty-launcher-5-thread-1) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 7235 INFO  (jetty-launcher-5-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@160d84b3{/solr,null,AVAILABLE}
   [junit4]   2> 7237 INFO  (jetty-launcher-5-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@49c9548{HTTP/1.1,[http/1.1]}{127.0.0.1:34713}
   [junit4]   2> 7237 INFO  (jetty-launcher-5-thread-1) [    ] o.e.j.s.Server Started @9118ms
   [junit4]   2> 7237 INFO  (jetty-launcher-5-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=34713}
   [junit4]   2> 7237 ERROR (jetty-launcher-5-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 7237 INFO  (jetty-launcher-5-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.2.0
   [junit4]   2> 7238 INFO  (jetty-launcher-5-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 7238 INFO  (jetty-launcher-5-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 7238 INFO  (jetty-launcher-5-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-11-18T13:16:42.603Z
   [junit4]   2> 7238 INFO  (jetty-launcher-5-thread-1) [    ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 7244 INFO  (jetty-launcher-5-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 7268 INFO  (jetty-launcher-5-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40041/solr
   [junit4]   2> 7460 INFO  (jetty-launcher-5-thread-1) [n:127.0.0.1:34713_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 7461 INFO  (jetty-launcher-5-thread-1) [n:127.0.0.1:34713_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:34713_solr
   [junit4]   2> 7463 INFO  (jetty-launcher-5-thread-1) [n:127.0.0.1:34713_solr    ] o.a.s.c.Overseer Overseer (id=99025617057611779-127.0.0.1:34713_solr-n_0000000000) starting
   [junit4]   2> 7516 INFO  (jetty-launcher-5-thread-1) [n:127.0.0.1:34713_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:34713_solr
   [junit4]   2> 7529 INFO  (zkCallback-12-thread-1-processing-n:127.0.0.1:34713_solr) [n:127.0.0.1:34713_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 7669 INFO  (jetty-launcher-5-thread-1) [n:127.0.0.1:34713_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_34713.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7f4ce3dc
   [junit4]   2> 7696 INFO  (jetty-launcher-5-thread-1) [n:127.0.0.1:34713_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_34713.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7f4ce3dc
   [junit4]   2> 7696 INFO  (jetty-launcher-5-thread-1) [n:127.0.0.1:34713_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_34713.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7f4ce3dc
   [junit4]   2> 7719 INFO  (jetty-launcher-5-thread-1) [n:127.0.0.1:34713_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionReloadTest_30CBC7BAB2D589AA-001/tempDir-001/node1/.
   [junit4]   2> 7798 INFO  (SUITE-CollectionReloadTest-seed#[30CBC7BAB2D589AA]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 7799 INFO  (SUITE-CollectionReloadTest-seed#[30CBC7BAB2D589AA]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40041/solr ready
   [junit4]   2> 7815 INFO  (TEST-CollectionReloadTest.testReloadedLeaderStateAfterZkSessionLoss-seed#[30CBC7BAB2D589AA]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testReloadedLeaderStateAfterZkSessionLoss
   [junit4]   2> 7815 INFO  (TEST-CollectionReloadTest.testReloadedLeaderStateAfterZkSessionLoss-seed#[30CBC7BAB2D589AA]) [    ] o.a.s.c.CollectionReloadTest testReloadedLeaderStateAfterZkSessionLoss initialized OK ... running test logic
   [junit4]   2> 7837 INFO  (qtp1765126627-61) [n:127.0.0.1:34713_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf&name=c8n_1x1&nrtReplicas=1&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 7850 INFO  (OverseerThreadFactory-34-thread-1-processing-n:127.0.0.1:34713_solr) [n:127.0.0.1:34713_solr    ] o.a.s.c.CreateCollectionCmd Create collection c8n_1x1
   [junit4]   2> 7974 INFO  (OverseerStateUpdate-99025617057611779-127.0.0.1:34713_solr-n_0000000000) [n:127.0.0.1:34713_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"c8n_1x1",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"c8n_1x1_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:34713/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 8187 INFO  (qtp1765126627-63) [n:127.0.0.1:34713_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node2&collection.configName=conf&newCollection=true&name=c8n_1x1_shard1_replica_n1&action=CREATE&numShards=1&collection=c8n_1x1&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 8189 INFO  (qtp1765126627-63) [n:127.0.0.1:34713_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 8302 INFO  (zkCallback-12-thread-1-processing-n:127.0.0.1:34713_solr) [n:127.0.0.1:34713_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/c8n_1x1/state.json] for collection [c8n_1x1] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 8302 INFO  (zkCallback-12-thread-2-processing-n:127.0.0.1:34713_solr) [n:127.0.0.1:34713_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/c8n_1x1/state.json] for collection [c8n_1x1] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 9208 INFO  (qtp1765126627-63) [n:127.0.0.1:34713_solr c:c8n_1x1 s:shard1 r:core_node2 x:c8n_1x1_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
   [junit4]   2> 9215 INFO  (qtp1765126627-63) [n:127.0.0.1:34713_solr c:c8n_1x1 s:shard1 r:core_node2 x:c8n_1x1_shard1_replica_n1] o.a.s.s.IndexSchema [c8n_1x1_shard1_replica_n1] Schema name=minimal
   [junit4]   2> 9219 INFO  (qtp1765126627-63) [n:127.0.0.1:34713_solr c:c8n_1x1 s:shard1 r:core_node2 x:c8n_1x1_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 9219 INFO  (qtp1765126627-63) [n:127.0.0.1:34713_solr c:c8n_1x1 s:shard1 r:core_node2 x:c8n_1x1_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'c8n_1x1_shard1_replica_n1' using configuration from collection c8n_1x1, trusted=true
   [junit4]   2> 9220 INFO  (qtp1765126627-63) [n:127.0.0.1:34713_solr c:c8n_1x1 s:shard1 r:core_node2 x:c8n_1x1_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_34713.solr.core.c8n_1x1.shard1.replica_n1' (registry 'solr.core.c8n_1x1.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7f4ce3dc
   [junit4]   2> 9221 INFO  (qtp1765126627-63) [n:127.0.0.1:34713_solr c:c8n_1x1 s:shard1 r:core_node2 x:c8n_1x1_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 9221 INFO  (qtp1765126627-63) [n:127.0.0.1:34713_solr c:c8n_1x1 s:shard1 r:core_node2 x:c8n_1x1_shard1_replica_n1] o.a.s.c.SolrCore [[c8n_1x1_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionReloadTest_30CBC7BAB2D589AA-001/tempDir-001/node1/c8n_1x1_shard1_replica_n1], dataDir=[/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionReloadTest_30CBC7BAB2D589AA-001/tempDir-001/node1/./c8n_1x1_shard1_replica_n1/data/]
   [junit4]   2> 9276 INFO  (qtp1765126627-63) [n:127.0.0.1:34713_solr c:c8n_1x1 s:shard1 r:core_node2 x:c8n_1x1_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 9276 INFO  (qtp1765126627-63) [n:127.0.0.1:34713_solr c:c8n_1x1 s:shard1 r:core_node2 x:c8n_1x1_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 9278 INFO  (qtp1765126627-63) [n:127.0.0.1:34713_solr c:c8n_1x1 s:shard1 r:core_node2 x:c8n_1x1_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 9278 INFO  (qtp1765126627-63) [n:127.0.0.1:34713_solr c:c8n_1x1 s:shard1 r:core_node2 x:c8n_1x1_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 9281 INFO  (qtp1765126627-63) [n:127.0.0.1:34713_solr c:c8n_1x1 s:shard1 r:core_node2 x:c8n_1x1_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6a8d716d[c8n_1x1_shard1_replica_n1] main]
   [junit4]   2> 9284 INFO  (qtp1765126627-63) [n:127.0.0.1:34713_solr c:c8n_1x1 s:shard1 r:core_node2 x:c8n_1x1_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 9285 INFO  (qtp1765126627-63) [n:127.0.0.1:34713_solr c:c8n_1x1 s:shard1 r:core_node2 x:c8n_1x1_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 9286 INFO  (qtp1765126627-63) [n:127.0.0.1:34713_solr c:c8n_1x1 s:shard1 r:core_node2 x:c8n_1x1_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 9289 INFO  (searcherExecutor-37-thread-1-processing-n:127.0.0.1:34713_solr x:c8n_1x1_shard1_replica_n1 s:shard1 c:c8n_1x1 r:core_node2) [n:127.0.0.1:34713_solr c:c8n_1x1 s:shard1 r:core_node2 x:c8n_1x1_shard1_replica_n1] o.a.s.c.SolrCore [c8n_1x1_shard1_replica_n1] Registered new searcher Searcher@6a8d716d[c8n_1x1_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 9290 INFO  (qtp1765126627-63) [n:127.0.0.1:34713_solr c:c8n_1x1 s:shard1 r:core_node2 x:c8n_1x1_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1584409875217121280
   [junit4]   2> 9302 INFO  (qtp1765126627-63) [n:127.0.0.1:34713_solr c:c8n_1x1 s:shard1 r:core_node2 x:c8n_1x1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 9302 INFO  (qtp1765126627-63) [n:127.0.0.1:34713_solr c:c8n_1x1 s:shard1 r:core_node2 x:c8n_1x1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 9303 INFO  (qtp1765126627-63) [n:127.0.0.1:34713_solr c:c8n_1x1 s:shard1 r:core_node2 x:c8n_1x1_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:34713/solr/c8n_1x1_shard1_replica_n1/
   [junit4]   2> 9304 INFO  (qtp1765126627-63) [n:127.0.0.1:34713_solr c:c8n_1x1 s:shard1 r:core_node2 x:c8n_1x1_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 9304 INFO  (qtp1765126627-63) [n:127.0.0.1:34713_solr c:c8n_1x1 s:shard1 r:core_node2 x:c8n_1x1_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:34713/solr/c8n_1x1_shard1_replica_n1/ has no replicas
   [junit4]   2> 9304 INFO  (qtp1765126627-63) [n:127.0.0.1:34713_solr c:c8n_1x1 s:shard1 r:core_node2 x:c8n_1x1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 9315 INFO  (qtp1765126627-63) [n:127.0.0.1:34713_solr c:c8n_1x1 s:shard1 r:core_node2 x:c8n_1x1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:34713/solr/c8n_1x1_shard1_replica_n1/ shard1
   [junit4]   2> 187847 ERROR (qtp1765126627-61) [n:127.0.0.1:34713_solr    ] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: create the collection time out:180s
   [junit4]   2> 	at org.apache.solr.handler.admin.CollectionsHandler.handleResponse(CollectionsHandler.java:312)
   [junit4]   2> 	at org.apache.solr.handler.admin.CollectionsHandler.invokeAction(CollectionsHandler.java:244)
   [junit4]   2> 	at org.apache.solr.handler.admin.CollectionsHandler.handleRequestBody(CollectionsHandler.java:222)
   [junit4]   2> 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:735)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:716)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:497)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:493)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 
   [junit4]   2> 187847 INFO  (qtp1765126627-61) [n:127.0.0.1:34713_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf&name=c8n_1x1&nrtReplicas=1&action=CREATE&numShards=1&wt=javabin&version=2} status=500 QTime=180013
   [junit4]   2> 187848 ERROR (qtp1765126627-61) [n:127.0.0.1:34713_solr    ] o.a.s.s.HttpSolrCall null:org.apache.solr.common.SolrException: create the collection time out:180s
   [junit4]   2> 	at org.apache.solr.handler.admin.CollectionsHandler.handleResponse(CollectionsHandler.java:312)
   [junit4]   2> 	at org.apache.solr.handler.admin.CollectionsHandler.invokeAction(CollectionsHandler.java:244)
   [junit4]   2> 	at org.apache.solr.handler.admin.CollectionsHandler.handleRequestBody(CollectionsHandler.java:222)
   [junit4]   2> 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:735)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:716)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:497)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:493)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 
   [junit4]   2> 187855 INFO  (TEST-CollectionReloadTest.testReloadedLeaderStateAfterZkSessionLoss-seed#[30CBC7BAB2D589AA]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testReloadedLeaderStateAfterZkSessionLoss
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=CollectionReloadTest -Dtests.method=testReloadedLeaderStateAfterZkSessionLoss -Dtests.seed=30CBC7BAB2D589AA -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=sr-Latn-ME -Dtests.timezone=Asia/Muscat -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR    180s J2 | CollectionReloadTest.testReloadedLeaderStateAfterZkSessionLoss <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:34713/solr: create the collection time out:180s
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([30CBC7BAB2D589AA:CBE57CE71438FAFF]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1103)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:883)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:816)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
   [junit4]    > 	at org.apache.solr.cloud.CollectionReloadTest.testReloadedLeaderStateAfterZkSessionLoss(CollectionReloadTest.java:53)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 187861 INFO  (jetty-closer-6-thread-1) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@49c9548{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 187862 INFO  (jetty-closer-6-thread-1) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=36692604
   [junit4]   2> 187862 INFO  (jetty-closer-6-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 187862 INFO  (jetty-closer-6-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@37543996: rootName = solr_34713, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@675f1059
   [junit4]   2> 187863 ERROR (OverseerThreadFactory-34-thread-1-processing-n:127.0.0.1:34713_solr) [n:127.0.0.1:34713_solr    ] o.a.s.c.OverseerCollectionMessageHandler Error from shard: http://127.0.0.1:34713/solr
   [junit4]   2> org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:34713/solr
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:657)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
   [junit4]   2> 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
   [junit4]   2> 	at org.apache.solr.handler.component.HttpShardHandler.lambda$submit$0(HttpShardHandler.java:172)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:34713 failed to respond
   [junit4]   2> 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:141)
   [junit4]   2> 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
   [junit4]   2> 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
   [junit4]   2> 	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
   [junit4]   2> 	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
   [junit4]   2> 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
   [junit4]   2> 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
   [junit4]   2> 	at org.apache.solr.util.stats.InstrumentedHttpRequestExecutor.execute(InstrumentedHttpRequestExecutor.java:118)
   [junit4]   2> 	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
   [junit4]   2> 	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
   [junit4]   2> 	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
   [junit4]   2> 	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
   [junit4]   2> 	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
   [junit4]   2> 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
   [junit4]   2> 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:542)
   [junit4]   2> 	... 12 more
   [junit4]   2> 187865 ERROR (OverseerThreadFactory-34-thread-1-processing-n:127.0.0.1:34713_solr) [n:127.0.0.1:34713_solr    ] o.a.s.c.OverseerCollectionMessageHandler Cleaning up collection [c8n_1x1].
   [junit4]   2> 187870 INFO  (OverseerThreadFactory-34-thread-1-processing-n:127.0.0.1:34713_solr) [n:127.0.0.1:34713_solr    ] o.a.s.c.OverseerCollectionMessageHandler Executing Collection Cmd : action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true
   [junit4]   2> 187872 ERROR (OverseerThreadFactory-34-thread-1-processing-n:127.0.0.1:34713_solr) [n:127.0.0.1:34713_solr    ] o.a.s.c.OverseerCollectionMessageHandler Error from shard: http://127.0.0.1:34713/solr
   [junit4]   2> org.apache.solr.client.solrj.SolrServerException: Server refused connection at: http://127.0.0.1:34713/solr
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:650)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
   [junit4]   2> 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
   [junit4]   2> 	at org.apache.solr.handler.component.HttpShardHandler.lambda$submit$0(HttpShardHandler.java:172)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: org.apache.http.conn.HttpHostConnectException: Connect to 127.0.0.1:34713 [/127.0.0.1] failed: Connection refused (Connection refused)
   [junit4]   2> 	at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:159)
   [junit4]   2> 	at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:359)
   [junit4]   2> 	at org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:381)
   [junit4]   2> 	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:237)
   [junit4]   2> 	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
   [junit4]   2> 	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
   [junit4]   2> 	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
   [junit4]   2> 	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
   [junit4]   2> 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
   [junit4]   2> 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:542)
   [junit4]   2> 	... 12 more
   [junit4]   2> Caused by: java.net.ConnectException: Connection refused (Connection refused)
   [junit4]   2> 	at java.net.PlainSocketImpl.socketConnect(Native Method)
   [junit4]   2> 	at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
   [junit4]   2> 	at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
   [junit4]   2> 	at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
   [junit4]   2> 	at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
   [junit4]   2> 	at java.net.Socket.connect(Socket.java:589)
   [junit4]   2> 	at org.apache.http.conn.socket.PlainConnectionSocketFactory.connectSocket(PlainConnectionSocketFactory.java:75)
   [junit4]   2> 	at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:142)
   [junit4]   2> 	... 22 more
   [junit4]   2> 187885 INFO  (jetty-closer-6-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 187885 INFO  (jetty-closer-6-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@35191ae6: rootName = solr_34713, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@6998e2c2
   [junit4]   2> 187893 INFO  (jetty-closer-6-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 187894 INFO  (jetty-closer-6-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@6bd84fd1: rootName = solr_34713, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@6da77b29
   [junit4]   2> 187895 INFO  (jetty-closer-6-thread-1) [    ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:34713_solr
   [junit4]   2> 187896 INFO  (jetty-closer-6-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 187896 INFO  (zkCallback-12-thread-5-processing-n:127.0.0.1:34713_solr) [n:127.0.0.1:34713_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
   [junit4]   2> 187896 INFO  (coreCloseExecutor-42-thread-1) [n:127.0.0.1:34713_solr c:c8n_1x1 s:shard1 r:core_node2 x:c8n_1x1_shard1_replica_n1] o.a.s.c.SolrCore [c8n_1x1_shard1_replica_n1]  CLOSING SolrCore org.apache.solr.core.SolrCore@505a5b43
   [junit4]   2> 187896 INFO  (coreCloseExecutor-42-thread-1) [n:127.0.0.1:34713_solr c:c8n_1x1 s:shard1 r:core_node2 x:c8n_1x1_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.c8n_1x1.shard1.replica_n1, tag=1348098883
   [junit4]   2> 187896 INFO  (coreCloseExecutor-42-thread-1) [n:127.0.0.1:34713_solr c:c8n_1x1 s:shard1 r:core_node2 x:c8n_1x1_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@70b274f8: rootName = solr_34713, domain = solr.core.c8n_1x1.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.c8n_1x1.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@54d3fdab
   [junit4]   2> 187910 INFO  (coreCloseExecutor-42-thread-1) [n:127.0.0.1:34713_solr c:c8n_1x1 s:shard1 r:core_node2 x:c8n_1x1_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.c8n_1x1.shard1.leader, tag=1348098883
   [junit4]   2> 187914 INFO  (jetty-closer-6-thread-1) [    ] o.a.s.c.Overseer Overseer (id=99025617057611779-127.0.0.1:34713_solr-n_0000000000) closing
   [junit4]   2> 187914 INFO  (OverseerStateUpdate-99025617057611779-127.0.0.1:34713_solr-n_0000000000) [n:127.0.0.1:34713_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:34713_solr
   [junit4]   2> 217909 INFO  (zkCallback-12-thread-5-processing-n:127.0.0.1:34713_solr) [n:127.0.0.1:34713_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDeleted path:/collections/c8n_1x1/state.json] for collection [c8n_1x1] has occurred - updating... (live nodes size: [0])
   [junit4]   2> 217909 INFO  (zkCallback-12-thread-4-processing-n:127.0.0.1:34713_solr) [n:127.0.0.1:34713_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDeleted path:/collections/c8n_1x1/state.json] for collection [c8n_1x1] has occurred - updating... (live nodes size: [0])
   [junit4]   2> 217912 ERROR (OverseerThreadFactory-34-thread-1-processing-n:127.0.0.1:34713_solr) [n:127.0.0.1:34713_solr    ] o.a.s.c.OverseerCollectionMessageHandler Collection: c8n_1x1 operation: create failed:org.apache.solr.common.SolrException: Could not fully remove collection: c8n_1x1
   [junit4]   2> 	at org.apache.solr.cloud.DeleteCollectionCmd.call(DeleteCollectionCmd.java:109)
   [junit4]   2> 	at org.apache.solr.cloud.OverseerCollectionMessageHandler.cleanupCollection(OverseerCollectionMessageHandler.java:678)
   [junit4]   2> 	at org.apache.solr.cloud.CreateCollectionCmd.call(CreateCollectionCmd.java:302)
   [junit4]   2> 	at org.apache.solr.cloud.OverseerCollectionMessageHandler.processMessage(OverseerCollectionMessageHandler.java:232)
   [junit4]   2> 	at org.apache.solr.cloud.OverseerTaskProcessor$Runner.run(OverseerTaskProcessor.java:464)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 
   [junit4]   2> 217925 INFO  (OverseerThreadFactory-34-thread-1-processing-n:127.0.0.1:34713_solr) [n:127.0.0.1:34713_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 217925 WARN  (OverseerAutoScalingTriggerThread-99025617057611779-127.0.0.1:34713_solr-n_0000000000) [n:127.0.0.1:34713_solr    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 217927 INFO  (jetty-closer-6-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@160d84b3{/solr,null,UNAVAILABLE}
   [junit4]   2> 217948 ERROR (qtp1765126627-63) [n:127.0.0.1:34713_solr c:c8n_1x1 s:shard1 r:core_node2 x:c8n_1x1_shard1_replica_n1] o.a.s.c.ZkController Error getting leader from zk
   [junit4]   2> org.apache.solr.common.SolrException: No registered leader was found after waiting for 1220000ms , collection: c8n_1x1 slice: shard1 saw state=null with live_nodes=[]
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:771)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:736)
   [junit4]   2> 	at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1178)
   [junit4]   2> 	at org.apache.solr.cloud.ZkController.register(ZkController.java:1079)
   [junit4]   2> 	at org.apache.solr.cloud.ZkController.register(ZkController.java:1014)
   [junit4]   2> 	at org.apache.solr.core.ZkContainer.lambda$registerInZk$0(ZkContainer.java:181)
   [junit4]   2> 	at org.apache.solr.core.ZkContainer.registerInZk(ZkContainer.java:208)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.registerCore(CoreContainer.java:886)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1047)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:948)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
   [junit4]   2> 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:735)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:716)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:497)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 217949 ERROR (qtp1765126627-63) [n:127.0.0.1:34713_solr c:c8n_1x1 s:shard1 r:core_node2 x:c8n_1x1_shard1_replica_n1] o.a.s.c.ZkContainer :org.apache.solr.common.SolrException: Error getting leader from zk for shard shard1
   [junit4]   2> 	at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1207)
   [junit4]   2> 	at org.apache.solr.cloud.ZkController.register(ZkController.java:1079)
   [junit4]   2> 	at org.apache.solr.cloud.ZkController.register(ZkController.java:1014)
   [junit4]   2> 	at org.apache.solr.core.ZkContainer.lambda$registerInZk$0(ZkContainer.java:181)
   [junit4]   2> 	at org.apache.solr.core.ZkContainer.registerInZk(ZkContainer.java:208)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.registerCore(CoreContainer.java:886)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1047)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:948)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
   [junit4]   2> 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:735)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:716)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:497)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: No registered leader was found after waiting for 1220000ms , collection: c8n_1x1 slice: shard1 saw state=null with live_nodes=[]
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:771)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:736)
   [junit4]   2> 	at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1178)
   [junit4]   2> 	... 41 more
   [junit4]   2> 
   [junit4]   2> 217958 INFO  (qtp1765126627-63) [n:127.0.0.1:34713_solr c:c8n_1x1 s:shard1 r:core_node2 x:c8n_1x1_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node2&collection.configName=conf&newCollection=true&name=c8n_1x1_shard1_replica_n1&action=CREATE&numShards=1&collection=c8n_1x1&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=209772
   [junit4]   2> 217959 ERROR (SUITE-CollectionReloadTest-seed#[30CBC7BAB2D589AA]-worker) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 217959 INFO  (SUITE-CollectionReloadTest-seed#[30CBC7BAB2D589AA]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:40041 40041
   [junit4]   2> 228085 INFO  (Thread-12) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:40041 40041
   [junit4]   2> 228086 WARN  (Thread-12) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	2	/solr/aliases.json
   [junit4]   2> 	2	/solr/clusterprops.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	3	/solr/collections/c8n_1x1/state.json
   [junit4]   2> 	2	/solr/clusterstate.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	2	/solr/live_nodes
   [junit4]   2> 	2	/solr/collections
   [junit4]   2> 
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionReloadTest_30CBC7BAB2D589AA-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=1209, maxMBSortInHeap=7.532346237136817, sim=RandomSimilarity(queryNorm=true): {}, locale=sr-Latn-ME, timezone=Asia/Muscat
   [junit4]   2> NOTE: Linux 4.10.0-37-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=8,threads=1,free=268078192,total=417857536
   [junit4]   2> NOTE: All tests run in this JVM: [JSONWriterTest, TestCopyFieldCollectionResource, CollectionReloadTest]
   [junit4] Completed [23/749 (1!)] on J2 in 221.13s, 1 test, 1 error <<< FAILURES!

[...truncated 1169 lines...]
   [junit4] Suite: org.apache.solr.update.processor.TimePartitionedUpdateProcessorTest
   [junit4]   2> Creating dataDir: /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.update.processor.TimePartitionedUpdateProcessorTest_30CBC7BAB2D589AA-001/init-core-data-001
   [junit4]   2> 1337550 WARN  (SUITE-TimePartitionedUpdateProcessorTest-seed#[30CBC7BAB2D589AA]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=45 numCloses=45
   [junit4]   2> 1337550 INFO  (SUITE-TimePartitionedUpdateProcessorTest-seed#[30CBC7BAB2D589AA]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 1337551 INFO  (SUITE-TimePartitionedUpdateProcessorTest-seed#[30CBC7BAB2D589AA]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 1337552 INFO  (SUITE-TimePartitionedUpdateProcessorTest-seed#[30CBC7BAB2D589AA]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 1337552 INFO  (SUITE-TimePartitionedUpdateProcessorTest-seed#[30CBC7BAB2D589AA]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.update.processor.TimePartitionedUpdateProcessorTest_30CBC7BAB2D589AA-001/tempDir-001
   [junit4]   2> 1337552 INFO  (SUITE-TimePartitionedUpdateProcessorTest-seed#[30CBC7BAB2D589AA]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1337553 INFO  (Thread-2109) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1337553 INFO  (Thread-2109) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1337577 ERROR (Thread-2109) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 1337653 INFO  (SUITE-TimePartitionedUpdateProcessorTest-seed#[30CBC7BAB2D589AA]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:36861
   [junit4]   2> 1337663 INFO  (jetty-launcher-1439-thread-1) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1337663 INFO  (jetty-launcher-1439-thread-2) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1337665 INFO  (jetty-launcher-1439-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@33f5d821{/solr,null,AVAILABLE}
   [junit4]   2> 1337665 INFO  (jetty-launcher-1439-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@61bb603a{/solr,null,AVAILABLE}
   [junit4]   2> 1337672 INFO  (jetty-launcher-1439-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@1fe17378{SSL,[ssl, http/1.1]}{127.0.0.1:35801}
   [junit4]   2> 1337672 INFO  (jetty-launcher-1439-thread-1) [    ] o.e.j.s.Server Started @1339583ms
   [junit4]   2> 1337673 INFO  (jetty-launcher-1439-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=35801}
   [junit4]   2> 1337673 ERROR (jetty-launcher-1439-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1337673 INFO  (jetty-launcher-1439-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.2.0
   [junit4]   2> 1337673 INFO  (jetty-launcher-1439-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1337673 INFO  (jetty-launcher-1439-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1337673 INFO  (jetty-launcher-1439-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@6a8f29e7{SSL,[ssl, http/1.1]}{127.0.0.1:38435}
   [junit4]   2> 1337673 INFO  (jetty-launcher-1439-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-11-18T13:38:53.062Z
   [junit4]   2> 1337673 INFO  (jetty-launcher-1439-thread-2) [    ] o.e.j.s.Server Started @1339583ms
   [junit4]   2> 1337673 INFO  (jetty-launcher-1439-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=38435}
   [junit4]   2> 1337673 ERROR (jetty-launcher-1439-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1337674 INFO  (jetty-launcher-1439-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.2.0
   [junit4]   2> 1337674 INFO  (jetty-launcher-1439-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1337674 INFO  (jetty-launcher-1439-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1337674 INFO  (jetty-launcher-1439-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-11-18T13:38:53.063Z
   [junit4]   2> 1337678 INFO  (jetty-launcher-1439-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1337678 INFO  (jetty-launcher-1439-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1337690 INFO  (jetty-launcher-1439-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36861/solr
   [junit4]   2> 1337691 INFO  (jetty-launcher-1439-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36861/solr
   [junit4]   2> 1337759 INFO  (jetty-launcher-1439-thread-1) [n:127.0.0.1:35801_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1337759 INFO  (jetty-launcher-1439-thread-2) [n:127.0.0.1:38435_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1337759 INFO  (jetty-launcher-1439-thread-1) [n:127.0.0.1:35801_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:35801_solr
   [junit4]   2> 1337759 INFO  (jetty-launcher-1439-thread-1) [n:127.0.0.1:35801_solr    ] o.a.s.c.Overseer Overseer (id=99025704252145670-127.0.0.1:35801_solr-n_0000000000) starting
   [junit4]   2> 1337759 INFO  (jetty-launcher-1439-thread-2) [n:127.0.0.1:38435_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38435_solr
   [junit4]   2> 1337769 INFO  (zkCallback-1451-thread-1-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1337772 INFO  (zkCallback-1450-thread-1-processing-n:127.0.0.1:38435_solr) [n:127.0.0.1:38435_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1337775 INFO  (jetty-launcher-1439-thread-1) [n:127.0.0.1:35801_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35801_solr
   [junit4]   2> 1337780 INFO  (zkCallback-1450-thread-1-processing-n:127.0.0.1:38435_solr) [n:127.0.0.1:38435_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1337784 INFO  (zkCallback-1451-thread-1-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1337830 INFO  (jetty-launcher-1439-thread-2) [n:127.0.0.1:38435_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38435.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@21630aab
   [junit4]   2> 1337831 INFO  (jetty-launcher-1439-thread-1) [n:127.0.0.1:35801_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35801.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@21630aab
   [junit4]   2> 1337835 INFO  (jetty-launcher-1439-thread-2) [n:127.0.0.1:38435_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38435.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@21630aab
   [junit4]   2> 1337835 INFO  (jetty-launcher-1439-thread-1) [n:127.0.0.1:35801_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35801.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@21630aab
   [junit4]   2> 1337836 INFO  (jetty-launcher-1439-thread-2) [n:127.0.0.1:38435_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38435.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@21630aab
   [junit4]   2> 1337836 INFO  (jetty-launcher-1439-thread-1) [n:127.0.0.1:35801_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35801.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@21630aab
   [junit4]   2> 1337836 INFO  (jetty-launcher-1439-thread-1) [n:127.0.0.1:35801_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.update.processor.TimePartitionedUpdateProcessorTest_30CBC7BAB2D589AA-001/tempDir-001/node1/.
   [junit4]   2> 1337836 INFO  (jetty-launcher-1439-thread-2) [n:127.0.0.1:38435_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.update.processor.TimePartitionedUpdateProcessorTest_30CBC7BAB2D589AA-001/tempDir-001/node2/.
   [junit4]   2> 1337898 INFO  (TEST-TimePartitionedUpdateProcessorTest.test-seed#[30CBC7BAB2D589AA]) [    ] o.a.s.SolrTestCaseJ4 ###Starting test
   [junit4]   2> 1337900 INFO  (TEST-TimePartitionedUpdateProcessorTest.test-seed#[30CBC7BAB2D589AA]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1337901 INFO  (TEST-TimePartitionedUpdateProcessorTest.test-seed#[30CBC7BAB2D589AA]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:36861/solr ready
   [junit4]   2> 1337923 INFO  (qtp1359454324-9563) [n:127.0.0.1:38435_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params pullReplicas=0&replicationFactor=1&name=timeConfig&nrtReplicas=1&action=CREATE&numShards=1&tlogReplicas=0&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1337926 INFO  (OverseerThreadFactory-4315-thread-1-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.CreateCollectionCmd Create collection timeConfig
   [junit4]   2> 1337926 INFO  (OverseerThreadFactory-4315-thread-1-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.CreateCollectionCmd Only _default config set found, using it.
   [junit4]   2> 1337926 INFO  (OverseerThreadFactory-4315-thread-1-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/protwords.txt to /configs/timeConfig/protwords.txt
   [junit4]   2> 1337927 INFO  (OverseerThreadFactory-4315-thread-1-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/managed-schema to /configs/timeConfig/managed-schema
   [junit4]   2> 1337927 INFO  (OverseerThreadFactory-4315-thread-1-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/solrconfig.xml to /configs/timeConfig/solrconfig.xml
   [junit4]   2> 1337956 INFO  (OverseerThreadFactory-4315-thread-1-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/synonyms.txt to /configs/timeConfig/synonyms.txt
   [junit4]   2> 1337956 INFO  (OverseerThreadFactory-4315-thread-1-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/stopwords.txt to /configs/timeConfig/stopwords.txt
   [junit4]   2> 1337957 INFO  (OverseerThreadFactory-4315-thread-1-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/contractions_it.txt to /configs/timeConfig/lang/contractions_it.txt
   [junit4]   2> 1337957 INFO  (OverseerThreadFactory-4315-thread-1-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stemdict_nl.txt to /configs/timeConfig/lang/stemdict_nl.txt
   [junit4]   2> 1337958 INFO  (OverseerThreadFactory-4315-thread-1-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_hy.txt to /configs/timeConfig/lang/stopwords_hy.txt
   [junit4]   2> 1337959 INFO  (OverseerThreadFactory-4315-thread-1-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_no.txt to /configs/timeConfig/lang/stopwords_no.txt
   [junit4]   2> 1337959 INFO  (OverseerThreadFactory-4315-thread-1-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/contractions_ca.txt to /configs/timeConfig/lang/contractions_ca.txt
   [junit4]   2> 1337960 INFO  (OverseerThreadFactory-4315-thread-1-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_id.txt to /configs/timeConfig/lang/stopwords_id.txt
   [junit4]   2> 1337960 INFO  (OverseerThreadFactory-4315-thread-1-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_ga.txt to /configs/timeConfig/lang/stopwords_ga.txt
   [junit4]   2> 1337961 INFO  (OverseerThreadFactory-4315-thread-1-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_hi.txt to /configs/timeConfig/lang/stopwords_hi.txt
   [junit4]   2> 1337961 INFO  (OverseerThreadFactory-4315-thread-1-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_da.txt to /configs/timeConfig/lang/stopwords_da.txt
   [junit4]   2> 1337962 INFO  (OverseerThreadFactory-4315-thread-1-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_ja.txt to /configs/timeConfig/lang/stopwords_ja.txt
   [junit4]   2> 1337962 INFO  (OverseerThreadFactory-4315-thread-1-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_pt.txt to /configs/timeConfig/lang/stopwords_pt.txt
   [junit4]   2> 1337963 INFO  (OverseerThreadFactory-4315-thread-1-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_hu.txt to /configs/timeConfig/lang/stopwords_hu.txt
   [junit4]   2> 1337963 INFO  (OverseerThreadFactory-4315-thread-1-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_el.txt to /configs/timeConfig/lang/stopwords_el.txt
   [junit4]   2> 1337964 INFO  (OverseerThreadFactory-4315-thread-1-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_ru.txt to /configs/timeConfig/lang/stopwords_ru.txt
   [junit4]   2> 1337965 INFO  (OverseerThreadFactory-4315-thread-1-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_tr.txt to /configs/timeConfig/lang/stopwords_tr.txt
   [junit4]   2> 1337965 INFO  (OverseerThreadFactory-4315-thread-1-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_ar.txt to /configs/timeConfig/lang/stopwords_ar.txt
   [junit4]   2> 1337966 INFO  (OverseerThreadFactory-4315-thread-1-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/userdict_ja.txt to /configs/timeConfig/lang/userdict_ja.txt
   [junit4]   2> 1337966 INFO  (OverseerThreadFactory-4315-thread-1-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_eu.txt to /configs/timeConfig/lang/stopwords_eu.txt
   [junit4]   2> 1337967 INFO  (OverseerThreadFactory-4315-thread-1-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_it.txt to /configs/timeConfig/lang/stopwords_it.txt
   [junit4]   2> 1337967 INFO  (OverseerThreadFactory-4315-thread-1-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_cz.txt to /configs/timeConfig/lang/stopwords_cz.txt
   [junit4]   2> 1337968 INFO  (OverseerThreadFactory-4315-thread-1-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stoptags_ja.txt to /configs/timeConfig/lang/stoptags_ja.txt
   [junit4]   2> 1337968 INFO  (OverseerThreadFactory-4315-thread-1-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/contractions_fr.txt to /configs/timeConfig/lang/contractions_fr.txt
   [junit4]   2> 1337969 INFO  (OverseerThreadFactory-4315-thread-1-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_de.txt to /configs/timeConfig/lang/stopwords_de.txt
   [junit4]   2> 1337969 INFO  (OverseerThreadFactory-4315-thread-1-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_fa.txt to /configs/timeConfig/lang/stopwords_fa.txt
   [junit4]   2> 1337970 INFO  (OverseerThreadFactory-4315-thread-1-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/contractions_ga.txt to /configs/timeConfig/lang/contractions_ga.txt
   [junit4]   2> 1337970 INFO  (OverseerThreadFactory-4315-thread-1-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_ca.txt to /configs/timeConfig/lang/stopwords_ca.txt
   [junit4]   2> 1337971 INFO  (OverseerThreadFactory-4315-thread-1-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_nl.txt to /configs/timeConfig/lang/stopwords_nl.txt
   [junit4]   2> 1337971 INFO  (OverseerThreadFactory-4315-thread-1-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_bg.txt to /configs/timeConfig/lang/stopwords_bg.txt
   [junit4]   2> 1337972 INFO  (OverseerThreadFactory-4315-thread-1-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_lv.txt to /configs/timeConfig/lang/stopwords_lv.txt
   [junit4]   2> 1337972 INFO  (OverseerThreadFactory-4315-thread-1-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_en.txt to /configs/timeConfig/lang/stopwords_en.txt
   [junit4]   2> 1337973 INFO  (OverseerThreadFactory-4315-thread-1-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_es.txt to /configs/timeConfig/lang/stopwords_es.txt
   [junit4]   2> 1337974 INFO  (OverseerThreadFactory-4315-thread-1-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_th.txt to /configs/timeConfig/lang/stopwords_th.txt
   [junit4]   2> 1337974 INFO  (OverseerThreadFactory-4315-thread-1-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_ro.txt to /configs/timeConfig/lang/stopwords_ro.txt
   [junit4]   2> 1337975 INFO  (OverseerThreadFactory-4315-thread-1-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_fr.txt to /configs/timeConfig/lang/stopwords_fr.txt
   [junit4]   2> 1337975 INFO  (OverseerThreadFactory-4315-thread-1-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_sv.txt to /configs/timeConfig/lang/stopwords_sv.txt
   [junit4]   2> 1337976 INFO  (OverseerThreadFactory-4315-thread-1-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_fi.txt to /configs/timeConfig/lang/stopwords_fi.txt
   [junit4]   2> 1337976 INFO  (OverseerThreadFactory-4315-thread-1-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/hyphenations_ga.txt to /configs/timeConfig/lang/hyphenations_ga.txt
   [junit4]   2> 1337976 INFO  (OverseerThreadFactory-4315-thread-1-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_gl.txt to /configs/timeConfig/lang/stopwords_gl.txt
   [junit4]   2> 1337977 INFO  (OverseerThreadFactory-4315-thread-1-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/params.json to /configs/timeConfig/params.json
   [junit4]   2> 1338080 INFO  (OverseerStateUpdate-99025704252145670-127.0.0.1:35801_solr-n_0000000000) [n:127.0.0.1:35801_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"timeConfig",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"timeConfig_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:35801/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 1338302 INFO  (qtp977507926-9556) [n:127.0.0.1:35801_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node2&collection.configName=timeConfig&newCollection=true&name=timeConfig_shard1_replica_n1&action=CREATE&numShards=1&collection=timeConfig&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1338302 INFO  (qtp977507926-9556) [n:127.0.0.1:35801_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1338405 INFO  (zkCallback-1451-thread-1-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/timeConfig/state.json] for collection [timeConfig] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1338405 INFO  (zkCallback-1451-thread-2-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/timeConfig/state.json] for collection [timeConfig] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1339312 INFO  (qtp977507926-9556) [n:127.0.0.1:35801_solr c:timeConfig s:shard1 r:core_node2 x:timeConfig_shard1_replica_n1] o.a.s.c.RequestParams conf resource params.json loaded . version : 0 
   [junit4]   2> 1339312 INFO  (qtp977507926-9556) [n:127.0.0.1:35801_solr c:timeConfig s:shard1 r:core_node2 x:timeConfig_shard1_replica_n1] o.a.s.c.RequestParams request params refreshed to version 0
   [junit4]   2> 1339312 WARN  (qtp977507926-9556) [n:127.0.0.1:35801_solr c:timeConfig s:shard1 r:core_node2 x:timeConfig_shard1_replica_n1] o.a.s.c.SolrConfig Couldn't add files from /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.update.processor.TimePartitionedUpdateProcessorTest_30CBC7BAB2D589AA-001/tempDir-001/node1/timeConfig_shard1_replica_n1/../../../../contrib/extraction/lib filtered by .*\.jar to classpath: /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.update.processor.TimePartitionedUpdateProcessorTest_30CBC7BAB2D589AA-001/tempDir-001/node1/timeConfig_shard1_replica_n1/../../../../contrib/extraction/lib
   [junit4]   2> 1339313 WARN  (qtp977507926-9556) [n:127.0.0.1:35801_solr c:timeConfig s:shard1 r:core_node2 x:timeConfig_shard1_replica_n1] o.a.s.c.SolrConfig Couldn't add files from /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.update.processor.TimePartitionedUpdateProcessorTest_30CBC7BAB2D589AA-001/tempDir-001/node1/timeConfig_shard1_replica_n1/../../../../dist filtered by solr-cell-\d.*\.jar to classpath: /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.update.processor.TimePartitionedUpdateProcessorTest_30CBC7BAB2D589AA-001/tempDir-001/node1/timeConfig_shard1_replica_n1/../../../../dist
   [junit4]   2> 1339313 WARN  (qtp977507926-9556) [n:127.0.0.1:35801_solr c:timeConfig s:shard1 r:core_node2 x:timeConfig_shard1_replica_n1] o.a.s.c.SolrConfig Couldn't add files from /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.update.processor.TimePartitionedUpdateProcessorTest_30CBC7BAB2D589AA-001/tempDir-001/node1/timeConfig_shard1_replica_n1/../../../../contrib/clustering/lib filtered by .*\.jar to classpath: /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.update.processor.TimePartitionedUpdateProcessorTest_30CBC7BAB2D589AA-001/tempDir-001/node1/timeConfig_shard1_replica_n1/../../../../contrib/clustering/lib
   [junit4]   2> 1339313 WARN  (qtp977507926-9556) [n:127.0.0.1:35801_solr c:timeConfig s:shard1 r:core_node2 x:timeConfig_shard1_replica_n1] o.a.s.c.SolrConfig Couldn't add files from /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.update.processor.TimePartitionedUpdateProcessorTest_30CBC7BAB2D589AA-001/tempDir-001/node1/timeConfig_shard1_replica_n1/../../../../dist filtered by solr-clustering-\d.*\.jar to classpath: /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.update.processor.TimePartitionedUpdateProcessorTest_30CBC7BAB2D589AA-001/tempDir-001/node1/timeConfig_shard1_replica_n1/../../../../dist
   [junit4]   2> 1339313 WARN  (qtp977507926-9556) [n:127.0.0.1:35801_solr c:timeConfig s:shard1 r:core_node2 x:timeConfig_shard1_replica_n1] o.a.s.c.SolrConfig Couldn't add files from /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.update.processor.TimePartitionedUpdateProcessorTest_30CBC7BAB2D589AA-001/tempDir-001/node1/timeConfig_shard1_replica_n1/../../../../contrib/langid/lib filtered by .*\.jar to classpath: /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.update.processor.TimePartitionedUpdateProcessorTest_30CBC7BAB2D589AA-001/tempDir-001/node1/timeConfig_shard1_replica_n1/../../../../contrib/langid/lib
   [junit4]   2> 1339313 WARN  (qtp977507926-9556) [n:127.0.0.1:35801_solr c:timeConfig s:shard1 r:core_node2 x:timeConfig_shard1_replica_n1] o.a.s.c.SolrConfig Couldn't add files from /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.update.processor.TimePartitionedUpdateProcessorTest_30CBC7BAB2D589AA-001/tempDir-001/node1/timeConfig_shard1_replica_n1/../../../../dist filtered by solr-langid-\d.*\.jar to classpath: /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.update.processor.TimePartitionedUpdateProcessorTest_30CBC7BAB2D589AA-001/tempDir-001/node1/timeConfig_shard1_replica_n1/../../../../dist
   [junit4]   2> 1339313 WARN  (qtp977507926-9556) [n:127.0.0.1:35801_solr c:timeConfig s:shard1 r:core_node2 x:timeConfig_shard1_replica_n1] o.a.s.c.SolrConfig Couldn't add files from /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.update.processor.TimePartitionedUpdateProcessorTest_30CBC7BAB2D589AA-001/tempDir-001/node1/timeConfig_shard1_replica_n1/../../../../contrib/velocity/lib filtered by .*\.jar to classpath: /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.update.processor.TimePartitionedUpdateProcessorTest_30CBC7BAB2D589AA-001/tempDir-001/node1/timeConfig_shard1_replica_n1/../../../../contrib/velocity/lib
   [junit4]   2> 1339314 WARN  (qtp977507926-9556) [n:127.0.0.1:35801_solr c:timeConfig s:shard1 r:core_node2 x:timeConfig_shard1_replica_n1] o.a.s.c.SolrConfig Couldn't add files from /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.update.processor.TimePartitionedUpdateProcessorTest_30CBC7BAB2D589AA-001/tempDir-001/n

[...truncated too long message...]

gent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@29147412
   [junit4]   2> 1344692 INFO  (jetty-closer-1440-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 1344692 INFO  (jetty-closer-1440-thread-2) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@2d63f96b: rootName = solr_35801, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@6b218156
   [junit4]   2> 1344693 INFO  (jetty-closer-1440-thread-2) [    ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:35801_solr
   [junit4]   2> 1344693 INFO  (jetty-closer-1440-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 1344693 INFO  (zkCallback-1451-thread-2-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
   [junit4]   2> 1344693 INFO  (zkCallback-1450-thread-1-processing-n:127.0.0.1:38435_solr) [n:127.0.0.1:38435_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
   [junit4]   2> 1344696 INFO  (jetty-closer-1440-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 1344696 INFO  (jetty-closer-1440-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@2363d08d: rootName = solr_38435, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@6b218156
   [junit4]   2> 1344696 INFO  (coreCloseExecutor-4336-thread-1) [n:127.0.0.1:35801_solr c:myalias_2017-10-23 s:shard1 r:core_node2 x:myalias_2017-10-23_shard1_replica_n1] o.a.s.c.SolrCore [myalias_2017-10-23_shard1_replica_n1]  CLOSING SolrCore org.apache.solr.core.SolrCore@506febc4
   [junit4]   2> 1344696 INFO  (coreCloseExecutor-4336-thread-1) [n:127.0.0.1:35801_solr c:myalias_2017-10-23 s:shard1 r:core_node2 x:myalias_2017-10-23_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.myalias_2017-10-23.shard1.replica_n1, tag=1349512132
   [junit4]   2> 1344696 INFO  (coreCloseExecutor-4336-thread-1) [n:127.0.0.1:35801_solr c:myalias_2017-10-23 s:shard1 r:core_node2 x:myalias_2017-10-23_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@14b7f3a4: rootName = solr_35801, domain = solr.core.myalias_2017-10-23.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.myalias_2017-10-23.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@55331828
   [junit4]   2> 1344696 INFO  (jetty-closer-1440-thread-1) [    ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:38435_solr
   [junit4]   2> 1344696 INFO  (jetty-closer-1440-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 1344696 INFO  (jetty-closer-1440-thread-1) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1344696 INFO  (zkCallback-1451-thread-2-processing-n:127.0.0.1:35801_solr) [n:127.0.0.1:35801_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
   [junit4]   2> 1344696 INFO  (zkCallback-1450-thread-1-processing-n:127.0.0.1:38435_solr) [n:127.0.0.1:38435_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
   [junit4]   2> 1344697 INFO  (jetty-closer-1440-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@33f5d821{/solr,null,UNAVAILABLE}
   [junit4]   2> 1344704 INFO  (coreCloseExecutor-4336-thread-1) [n:127.0.0.1:35801_solr c:myalias_2017-10-23 s:shard1 r:core_node2 x:myalias_2017-10-23_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.myalias_2017-10-23.shard1.leader, tag=1349512132
   [junit4]   2> 1344705 INFO  (jetty-closer-1440-thread-2) [    ] o.a.s.c.Overseer Overseer (id=99025704252145670-127.0.0.1:35801_solr-n_0000000000) closing
   [junit4]   2> 1344705 INFO  (OverseerStateUpdate-99025704252145670-127.0.0.1:35801_solr-n_0000000000) [n:127.0.0.1:35801_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:35801_solr
   [junit4]   2> 1344706 WARN  (OverseerAutoScalingTriggerThread-99025704252145670-127.0.0.1:35801_solr-n_0000000000) [n:127.0.0.1:35801_solr    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 1344738 INFO  (jetty-closer-1440-thread-2) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@61bb603a{/solr,null,UNAVAILABLE}
   [junit4]   2> 1344738 ERROR (SUITE-TimePartitionedUpdateProcessorTest-seed#[30CBC7BAB2D589AA]-worker) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 1344738 INFO  (SUITE-TimePartitionedUpdateProcessorTest-seed#[30CBC7BAB2D589AA]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:36861 36861
   [junit4]   2> 1344855 INFO  (Thread-2109) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:36861 36861
   [junit4]   2> 1344856 WARN  (Thread-2109) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	4	/solr/aliases.json
   [junit4]   2> 	4	/solr/configs/timeConfig
   [junit4]   2> 	3	/solr/configs/timeConfig/managed-schema
   [junit4]   2> 	2	/solr/security.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	5	/solr/collections/timeConfig/state.json
   [junit4]   2> 	5	/solr/collections/myalias_2017-10-23/state.json
   [junit4]   2> 	4	/solr/clusterstate.json
   [junit4]   2> 	4	/solr/clusterprops.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	4	/solr/live_nodes
   [junit4]   2> 	4	/solr/collections
   [junit4]   2> 
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.update.processor.TimePartitionedUpdateProcessorTest_30CBC7BAB2D589AA-001
   [junit4]   2> Nov 18, 2017 1:39:00 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=1476, maxMBSortInHeap=6.3368729907938945, sim=RandomSimilarity(queryNorm=true): {}, locale=nn-NO, timezone=CST6CDT
   [junit4]   2> NOTE: Linux 4.10.0-37-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=8,threads=1,free=289511024,total=529530880
   [junit4]   2> NOTE: All tests run in this JVM: [TestCSVLoader, LeaderFailoverAfterPartitionTest, CollectionsAPISolrJTest, CoreAdminRequestStatusTest, TestReversedWildcardFilterFactory, SyncSliceTest, DistributedMLTComponentTest, HdfsRecoverLeaseTest, TestBackupRepositoryFactory, TestLocalFSCloudBackupRestore, TestTlogReplica, TestUtils, ZkControllerTest, AnalysisAfterCoreReloadTest, TestEmbeddedSolrServerSchemaAPI, TestSubQueryTransformerDistrib, BasicZkTest, SolrShardReporterTest, TestSolrCloudWithHadoopAuthPlugin, SearchHandlerTest, SpellCheckComponentTest, OutOfBoxZkACLAndCredentialsProvidersTest, TestDFRSimilarityFactory, ResourceLoaderTest, TestPostingsSolrHighlighter, BasicDistributedZkTest, HdfsChaosMonkeySafeLeaderTest, TestLegacyField, ActionThrottleTest, HdfsUnloadDistributedZkTest, TestNumericTerms32, TestSuggestSpellingConverter, TestLegacyTerms, TestLeaderElectionWithEmptyReplica, SaslZkACLProviderTest, V2StandaloneTest, TestWordDelimiterFilterFactory, CleanupOldIndexTest, TestAuthorizationFramework, DateMathParserTest, TestPHPSerializedResponseWriter, DistributedQueryComponentOptimizationTest, TlogReplayBufferedWhileIndexingTest, TestBlobHandler, TestTestInjection, TestConfig, StatelessScriptUpdateProcessorFactoryTest, CdcrBidirectionalTest, TestFieldCacheVsDocValues, TestIndexingPerformance, BlockDirectoryTest, TestSolrFieldCacheBean, TestSolrConfigHandler, TestJsonFacetRefinement, TestHalfAndHalfDocValues, AnalyticsMergeStrategyTest, TermVectorComponentTest, ChaosMonkeyNothingIsSafeWithPullReplicasTest, TestSolrCoreSnapshots, TestHashQParserPlugin, SpatialFilterTest, ImplicitSnitchTest, TestPayloadScoreQParserPlugin, TestCustomStream, ClassificationUpdateProcessorTest, TestExactSharedStatsCache, TestSmileRequest, RAMDirectoryFactoryTest, BlobRepositoryCloudTest, HdfsSyncSliceTest, TestSolrCloudWithDelegationTokens, DistributedQueueTest, TestConfigSetProperties, DistributedDebugComponentTest, SimpleCollectionCreateDeleteTest, SpellCheckCollatorTest, ExitableDirectoryReaderTest, BasicAuthIntegrationTest, TestRecovery, TestInfoStreamLogging, CdcrReplicationHandlerTest, HLLUtilTest, DeleteInactiveReplicaTest, DistribCursorPagingTest, TestUpdate, TestRequestStatusCollectionAPI, ZkFailoverTest, TestCloudInspectUtil, TestQueryUtils, SortSpecParsingTest, OverseerModifyCollectionTest, TestStressCloudBlindAtomicUpdates, TestNoOpRegenerator, DataDrivenBlockJoinTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, CloudMLTQParserTest, TestCodecSupport, TestRandomDVFaceting, TestHighlightDedupGrouping, TestDistributedSearch, TestDocBasedVersionConstraints, SoftAutoCommitTest, TestChildDocTransformer, SolrPluginUtilsTest, AddBlockUpdateTest, TestSolrDeletionPolicy1, QueryParsingTest, AtomicUpdatesTest, SimpleFacetsTest, TimePartitionedUpdateProcessorTest]
   [junit4] Completed [366/749 (2!)] on J0 in 8.14s, 2 tests, 1 failure <<< FAILURES!

[...truncated 48611 lines...]

[JENKINS] Lucene-Solr-7.x-Linux (32bit/jdk1.8.0_144) - Build # 847 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Linux/847/
Java: 32bit/jdk1.8.0_144 -client -XX:+UseConcMarkSweepGC

6 tests failed.
FAILED:  org.apache.solr.cloud.BasicDistributedZkTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:41763

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:41763
	at __randomizedtesting.SeedInfo.seed([EEA86119200536F7:66FC5EC38EF95B0F]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:654)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1103)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:883)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:816)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:314)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:991)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:171)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:542)
	... 43 more


FAILED:  org.apache.solr.cloud.TestCloudRecovery.corruptedLogTest

Error Message:
Timeout waiting for all live and active

Stack Trace:
java.lang.AssertionError: Timeout waiting for all live and active
	at __randomizedtesting.SeedInfo.seed([EEA86119200536F7:6DDE3EEBF67C3856]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.cloud.TestCloudRecovery.corruptedLogTest(TestCloudRecovery.java:184)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.autoscaling.ExecutePlanActionTest.testIntegration

Error Message:
Timed out waiting for replicas of collection to be 2 again null Live Nodes: [127.0.0.1:40797_solr] Last available state: DocCollection(testIntegration//collections/testIntegration/state.json/10)={   "pullReplicas":"0",   "replicationFactor":"2",   "shards":{"shard1":{       "range":"80000000-7fffffff",       "state":"active",       "replicas":{         "core_node3":{           "core":"testIntegration_shard1_replica_n1",           "base_url":"http://127.0.0.1:40797/solr",           "node_name":"127.0.0.1:40797_solr",           "state":"active",           "type":"NRT"},         "core_node4":{           "core":"testIntegration_shard1_replica_n2",           "base_url":"http://127.0.0.1:33705/solr",           "node_name":"127.0.0.1:33705_solr",           "state":"down",           "type":"NRT"},         "core_node6":{           "core":"testIntegration_shard1_replica_n5",           "base_url":"http://127.0.0.1:40797/solr",           "node_name":"127.0.0.1:40797_solr",           "state":"down",           "type":"NRT"}}}},   "router":{"name":"compositeId"},   "maxShardsPerNode":"1",   "autoAddReplicas":"false",   "nrtReplicas":"2",   "tlogReplicas":"0"}

Stack Trace:
java.lang.AssertionError: Timed out waiting for replicas of collection to be 2 again
null
Live Nodes: [127.0.0.1:40797_solr]
Last available state: DocCollection(testIntegration//collections/testIntegration/state.json/10)={
  "pullReplicas":"0",
  "replicationFactor":"2",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node3":{
          "core":"testIntegration_shard1_replica_n1",
          "base_url":"http://127.0.0.1:40797/solr",
          "node_name":"127.0.0.1:40797_solr",
          "state":"active",
          "type":"NRT"},
        "core_node4":{
          "core":"testIntegration_shard1_replica_n2",
          "base_url":"http://127.0.0.1:33705/solr",
          "node_name":"127.0.0.1:33705_solr",
          "state":"down",
          "type":"NRT"},
        "core_node6":{
          "core":"testIntegration_shard1_replica_n5",
          "base_url":"http://127.0.0.1:40797/solr",
          "node_name":"127.0.0.1:40797_solr",
          "state":"down",
          "type":"NRT"}}}},
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"1",
  "autoAddReplicas":"false",
  "nrtReplicas":"2",
  "tlogReplicas":"0"}
	at __randomizedtesting.SeedInfo.seed([EEA86119200536F7:5EC96F35053A97D2]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:269)
	at org.apache.solr.cloud.autoscaling.ExecutePlanActionTest.testIntegration(ExecutePlanActionTest.java:209)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.autoscaling.ExecutePlanActionTest.testExecute

Error Message:
Error from server at http://127.0.0.1:39003/solr: Could not fully create collection: testExecute

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:39003/solr: Could not fully create collection: testExecute
	at __randomizedtesting.SeedInfo.seed([EEA86119200536F7:DF18731432F2C970]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1103)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:883)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:816)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
	at org.apache.solr.cloud.autoscaling.ExecutePlanActionTest.testExecute(ExecutePlanActionTest.java:93)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.metrics.reporters.solr.SolrShardReporterTest.test

Error Message:
Timeout occured while waiting response from server at: https://127.0.0.1:35689

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:35689
	at __randomizedtesting.SeedInfo.seed([EEA86119200536F7:66FC5EC38EF95B0F]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:654)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1103)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:883)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:816)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:314)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:991)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:171)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
	at sun.security.ssl.InputRecord.read(InputRecord.java:503)
	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:983)
	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:940)
	at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:542)
	... 43 more


FAILED:  org.apache.solr.update.processor.TimePartitionedUpdateProcessorTest.test

Error Message:
Error from server at http://127.0.0.1:45161/solr: Timed out waiting to see all replicas: [myalias_2017-10-23_shard1_replica_n1] in cluster state.

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:45161/solr: Timed out waiting to see all replicas: [myalias_2017-10-23_shard1_replica_n1] in cluster state.
	at __randomizedtesting.SeedInfo.seed([EEA86119200536F7:66FC5EC38EF95B0F]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1103)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:883)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:816)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
	at org.apache.solr.update.processor.TimePartitionedUpdateProcessorTest.test(TimePartitionedUpdateProcessorTest.java:107)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 12067 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestCloudRecovery
   [junit4]   2> 788253 INFO  (SUITE-TestCloudRecovery-seed#[EEA86119200536F7]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudRecovery_EEA86119200536F7-001/init-core-data-001
   [junit4]   2> 788254 WARN  (SUITE-TestCloudRecovery-seed#[EEA86119200536F7]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=4 numCloses=4
   [junit4]   2> 788254 INFO  (SUITE-TestCloudRecovery-seed#[EEA86119200536F7]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 788255 INFO  (SUITE-TestCloudRecovery-seed#[EEA86119200536F7]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 788256 INFO  (SUITE-TestCloudRecovery-seed#[EEA86119200536F7]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudRecovery_EEA86119200536F7-001/tempDir-001
   [junit4]   2> 788256 INFO  (SUITE-TestCloudRecovery-seed#[EEA86119200536F7]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 788256 INFO  (Thread-1706) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 788256 INFO  (Thread-1706) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 788258 ERROR (Thread-1706) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 788356 INFO  (SUITE-TestCloudRecovery-seed#[EEA86119200536F7]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:34509
   [junit4]   2> 788362 INFO  (jetty-launcher-1296-thread-2) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 788362 INFO  (jetty-launcher-1296-thread-1) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 788363 INFO  (jetty-launcher-1296-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@ce656a{/solr,null,AVAILABLE}
   [junit4]   2> 788363 INFO  (jetty-launcher-1296-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1578892{/solr,null,AVAILABLE}
   [junit4]   2> 788365 INFO  (jetty-launcher-1296-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@ba0e5d{SSL,[ssl, http/1.1]}{127.0.0.1:42471}
   [junit4]   2> 788365 INFO  (jetty-launcher-1296-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@56c49f{SSL,[ssl, http/1.1]}{127.0.0.1:45387}
   [junit4]   2> 788365 INFO  (jetty-launcher-1296-thread-1) [    ] o.e.j.s.Server Started @789745ms
   [junit4]   2> 788365 INFO  (jetty-launcher-1296-thread-2) [    ] o.e.j.s.Server Started @789745ms
   [junit4]   2> 788366 INFO  (jetty-launcher-1296-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=42471}
   [junit4]   2> 788366 INFO  (jetty-launcher-1296-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=45387}
   [junit4]   2> 788366 ERROR (jetty-launcher-1296-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 788366 ERROR (jetty-launcher-1296-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 788366 INFO  (jetty-launcher-1296-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solrâ„¢ version 7.2.0
   [junit4]   2> 788366 INFO  (jetty-launcher-1296-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solrâ„¢ version 7.2.0
   [junit4]   2> 788366 INFO  (jetty-launcher-1296-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 788366 INFO  (jetty-launcher-1296-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 788366 INFO  (jetty-launcher-1296-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 788366 INFO  (jetty-launcher-1296-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 788366 INFO  (jetty-launcher-1296-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-11-18T09:47:16.462Z
   [junit4]   2> 788366 INFO  (jetty-launcher-1296-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-11-18T09:47:16.462Z
   [junit4]   2> 788368 INFO  (jetty-launcher-1296-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 788368 INFO  (jetty-launcher-1296-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 788374 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15fce866a010002, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 788376 INFO  (jetty-launcher-1296-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34509/solr
   [junit4]   2> 788376 INFO  (jetty-launcher-1296-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34509/solr
   [junit4]   2> 788378 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15fce866a010004, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 788392 INFO  (jetty-launcher-1296-thread-2) [n:127.0.0.1:45387_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 788392 INFO  (jetty-launcher-1296-thread-2) [n:127.0.0.1:45387_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:45387_solr
   [junit4]   2> 788393 INFO  (jetty-launcher-1296-thread-2) [n:127.0.0.1:45387_solr    ] o.a.s.c.Overseer Overseer (id=99024793524830213-127.0.0.1:45387_solr-n_0000000000) starting
   [junit4]   2> 788397 INFO  (jetty-launcher-1296-thread-2) [n:127.0.0.1:45387_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45387_solr
   [junit4]   2> 788399 INFO  (OverseerStateUpdate-99024793524830213-127.0.0.1:45387_solr-n_0000000000) [n:127.0.0.1:45387_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 788417 INFO  (jetty-launcher-1296-thread-1) [n:127.0.0.1:42471_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 788418 INFO  (jetty-launcher-1296-thread-1) [n:127.0.0.1:42471_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 788419 INFO  (jetty-launcher-1296-thread-1) [n:127.0.0.1:42471_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:42471_solr
   [junit4]   2> 788420 INFO  (zkCallback-1307-thread-1-processing-n:127.0.0.1:45387_solr) [n:127.0.0.1:45387_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 788420 INFO  (zkCallback-1308-thread-1-processing-n:127.0.0.1:42471_solr) [n:127.0.0.1:42471_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 788502 INFO  (jetty-launcher-1296-thread-2) [n:127.0.0.1:45387_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45387.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@fdd228
   [junit4]   2> 788516 INFO  (jetty-launcher-1296-thread-2) [n:127.0.0.1:45387_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45387.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@fdd228
   [junit4]   2> 788516 INFO  (jetty-launcher-1296-thread-2) [n:127.0.0.1:45387_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45387.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@fdd228
   [junit4]   2> 788517 INFO  (jetty-launcher-1296-thread-2) [n:127.0.0.1:45387_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudRecovery_EEA86119200536F7-001/tempDir-001/node2/.
   [junit4]   2> 788606 INFO  (jetty-launcher-1296-thread-1) [n:127.0.0.1:42471_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_42471.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@fdd228
   [junit4]   2> 788615 INFO  (jetty-launcher-1296-thread-1) [n:127.0.0.1:42471_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_42471.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@fdd228
   [junit4]   2> 788616 INFO  (jetty-launcher-1296-thread-1) [n:127.0.0.1:42471_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_42471.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@fdd228
   [junit4]   2> 788617 INFO  (jetty-launcher-1296-thread-1) [n:127.0.0.1:42471_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudRecovery_EEA86119200536F7-001/tempDir-001/node1/.
   [junit4]   2> 788644 INFO  (SUITE-TestCloudRecovery-seed#[EEA86119200536F7]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 788644 INFO  (SUITE-TestCloudRecovery-seed#[EEA86119200536F7]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:34509/solr ready
   [junit4]   2> 788730 INFO  (qtp22959668-6788) [n:127.0.0.1:42471_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params pullReplicas=0&replicationFactor=2&collection.configName=config&maxShardsPerNode=2&name=collection1&nrtReplicas=2&action=CREATE&numShards=2&tlogReplicas=0&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 788733 INFO  (OverseerThreadFactory-3011-thread-1-processing-n:127.0.0.1:45387_solr) [n:127.0.0.1:45387_solr    ] o.a.s.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 788838 INFO  (OverseerStateUpdate-99024793524830213-127.0.0.1:45387_solr-n_0000000000) [n:127.0.0.1:45387_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"collection1",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"collection1_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:42471/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 788840 INFO  (OverseerStateUpdate-99024793524830213-127.0.0.1:45387_solr-n_0000000000) [n:127.0.0.1:45387_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"collection1",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"collection1_shard1_replica_n2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:45387/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 788841 INFO  (OverseerStateUpdate-99024793524830213-127.0.0.1:45387_solr-n_0000000000) [n:127.0.0.1:45387_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"collection1",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"collection1_shard2_replica_n4",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:42471/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 788842 INFO  (OverseerStateUpdate-99024793524830213-127.0.0.1:45387_solr-n_0000000000) [n:127.0.0.1:45387_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"collection1",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"collection1_shard2_replica_n6",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:45387/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 789052 INFO  (qtp22959668-6853) [n:127.0.0.1:42471_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node7&collection.configName=config&newCollection=true&name=collection1_shard2_replica_n4&action=CREATE&numShards=2&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 789052 INFO  (qtp22959668-6790) [n:127.0.0.1:42471_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=config&newCollection=true&name=collection1_shard1_replica_n1&action=CREATE&numShards=2&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 789053 INFO  (qtp22959668-6853) [n:127.0.0.1:42471_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 789149 INFO  (qtp13914217-6798) [n:127.0.0.1:45387_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node8&collection.configName=config&newCollection=true&name=collection1_shard2_replica_n6&action=CREATE&numShards=2&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 789150 INFO  (qtp13914217-6798) [n:127.0.0.1:45387_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 789151 INFO  (qtp13914217-6789) [n:127.0.0.1:45387_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node5&collection.configName=config&newCollection=true&name=collection1_shard1_replica_n2&action=CREATE&numShards=2&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 789254 INFO  (zkCallback-1307-thread-1-processing-n:127.0.0.1:45387_solr) [n:127.0.0.1:45387_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 789254 INFO  (zkCallback-1308-thread-1-processing-n:127.0.0.1:42471_solr) [n:127.0.0.1:42471_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 789254 INFO  (zkCallback-1307-thread-2-processing-n:127.0.0.1:45387_solr) [n:127.0.0.1:45387_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 790075 INFO  (qtp22959668-6853) [n:127.0.0.1:42471_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_n4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
   [junit4]   2> 790077 INFO  (qtp22959668-6790) [n:127.0.0.1:42471_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
   [junit4]   2> 790089 INFO  (qtp22959668-6790) [n:127.0.0.1:42471_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.s.IndexSchema [collection1_shard1_replica_n1] Schema name=minimal
   [junit4]   2> 790089 INFO  (qtp22959668-6853) [n:127.0.0.1:42471_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_n4] o.a.s.s.IndexSchema [collection1_shard2_replica_n4] Schema name=minimal
   [junit4]   2> 790093 INFO  (qtp22959668-6853) [n:127.0.0.1:42471_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_n4] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 790093 INFO  (qtp22959668-6790) [n:127.0.0.1:42471_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 790093 INFO  (qtp22959668-6853) [n:127.0.0.1:42471_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_n4] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_n4' using configuration from collection collection1, trusted=true
   [junit4]   2> 790093 INFO  (qtp22959668-6790) [n:127.0.0.1:42471_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n1' using configuration from collection collection1, trusted=true
   [junit4]   2> 790093 INFO  (qtp22959668-6790) [n:127.0.0.1:42471_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_42471.solr.core.collection1.shard1.replica_n1' (registry 'solr.core.collection1.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@fdd228
   [junit4]   2> 790093 INFO  (qtp22959668-6853) [n:127.0.0.1:42471_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_n4] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_42471.solr.core.collection1.shard2.replica_n4' (registry 'solr.core.collection1.shard2.replica_n4') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@fdd228
   [junit4]   2> 790094 INFO  (qtp22959668-6790) [n:127.0.0.1:42471_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 790094 INFO  (qtp22959668-6853) [n:127.0.0.1:42471_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_n4] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 790094 INFO  (qtp22959668-6853) [n:127.0.0.1:42471_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_n4] o.a.s.c.SolrCore [[collection1_shard2_replica_n4] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudRecovery_EEA86119200536F7-001/tempDir-001/node1/collection1_shard2_replica_n4], dataDir=[/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudRecovery_EEA86119200536F7-001/tempDir-001/node1/./collection1_shard2_replica_n4/data/]
   [junit4]   2> 790094 INFO  (qtp22959668-6790) [n:127.0.0.1:42471_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.SolrCore [[collection1_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudRecovery_EEA86119200536F7-001/tempDir-001/node1/collection1_shard1_replica_n1], dataDir=[/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudRecovery_EEA86119200536F7-001/tempDir-001/node1/./collection1_shard1_replica_n1/data/]
   [junit4]   2> 790166 INFO  (qtp13914217-6798) [n:127.0.0.1:45387_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_n6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
   [junit4]   2> 790166 INFO  (qtp13914217-6789) [n:127.0.0.1:45387_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_n2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
   [junit4]   2> 790175 INFO  (qtp13914217-6789) [n:127.0.0.1:45387_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_n2] o.a.s.s.IndexSchema [collection1_shard1_replica_n2] Schema name=minimal
   [junit4]   2> 790176 INFO  (qtp13914217-6798) [n:127.0.0.1:45387_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_n6] o.a.s.s.IndexSchema [collection1_shard2_replica_n6] Schema name=minimal
   [junit4]   2> 790178 INFO  (qtp13914217-6789) [n:127.0.0.1:45387_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_n2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 790178 INFO  (qtp13914217-6789) [n:127.0.0.1:45387_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_n2] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n2' using configuration from collection collection1, trusted=true
   [junit4]   2> 790179 INFO  (qtp13914217-6789) [n:127.0.0.1:45387_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_n2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45387.solr.core.collection1.shard1.replica_n2' (registry 'solr.core.collection1.shard1.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@fdd228
   [junit4]   2> 790179 INFO  (qtp13914217-6798) [n:127.0.0.1:45387_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_n6] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 790179 INFO  (qtp13914217-6789) [n:127.0.0.1:45387_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_n2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 790179 INFO  (qtp13914217-6798) [n:127.0.0.1:45387_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_n6] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_n6' using configuration from collection collection1, trusted=true
   [junit4]   2> 790179 INFO  (qtp13914217-6789) [n:127.0.0.1:45387_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_n2] o.a.s.c.SolrCore [[collection1_shard1_replica_n2] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudRecovery_EEA86119200536F7-001/tempDir-001/node2/collection1_shard1_replica_n2], dataDir=[/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudRecovery_EEA86119200536F7-001/tempDir-001/node2/./collection1_shard1_replica_n2/data/]
   [junit4]   2> 790179 INFO  (qtp13914217-6798) [n:127.0.0.1:45387_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_n6] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45387.solr.core.collection1.shard2.replica_n6' (registry 'solr.core.collection1.shard2.replica_n6') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@fdd228
   [junit4]   2> 790180 INFO  (qtp13914217-6798) [n:127.0.0.1:45387_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_n6] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 790180 INFO  (qtp13914217-6798) [n:127.0.0.1:45387_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_n6] o.a.s.c.SolrCore [[collection1_shard2_replica_n6] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudRecovery_EEA86119200536F7-001/tempDir-001/node2/collection1_shard2_replica_n6], dataDir=[/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudRecovery_EEA86119200536F7-001/tempDir-001/node2/./collection1_shard2_replica_n6/data/]
   [junit4]   2> 790189 INFO  (qtp22959668-6790) [n:127.0.0.1:42471_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 790189 INFO  (qtp22959668-6790) [n:127.0.0.1:42471_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 790190 INFO  (qtp22959668-6790) [n:127.0.0.1:42471_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 790190 INFO  (qtp22959668-6790) [n:127.0.0.1:42471_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 790191 INFO  (qtp22959668-6853) [n:127.0.0.1:42471_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_n4] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 790191 INFO  (qtp22959668-6853) [n:127.0.0.1:42471_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_n4] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 790192 INFO  (qtp22959668-6853) [n:127.0.0.1:42471_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_n4] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 790192 INFO  (qtp22959668-6853) [n:127.0.0.1:42471_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_n4] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 790193 INFO  (qtp22959668-6790) [n:127.0.0.1:42471_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@e9f7[collection1_shard1_replica_n1] main]
   [junit4]   2> 790194 INFO  (qtp22959668-6790) [n:127.0.0.1:42471_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/config
   [junit4]   2> 790194 INFO  (qtp22959668-6790) [n:127.0.0.1:42471_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/config
   [junit4]   2> 790194 INFO  (qtp22959668-6853) [n:127.0.0.1:42471_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_n4] o.a.s.s.SolrIndexSearcher Opening [Searcher@176449d[collection1_shard2_replica_n4] main]
   [junit4]   2> 790195 INFO  (qtp22959668-6790) [n:127.0.0.1:42471_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 790195 INFO  (qtp22959668-6853) [n:127.0.0.1:42471_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_n4] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/config
   [junit4]   2> 790195 INFO  (qtp22959668-6853) [n:127.0.0.1:42471_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_n4] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/config
   [junit4]   2> 790196 INFO  (searcherExecutor-3017-thread-1-processing-n:127.0.0.1:42471_solr x:collection1_shard1_replica_n1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:42471_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.SolrCore [collection1_shard1_replica_n1] Registered new searcher Searcher@e9f7[collection1_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 790196 INFO  (qtp22959668-6790) [n:127.0.0.1:42471_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1584396698430472192
   [junit4]   2> 790196 INFO  (qtp22959668-6853) [n:127.0.0.1:42471_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_n4] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 790198 INFO  (searcherExecutor-3016-thread-1-processing-n:127.0.0.1:42471_solr x:collection1_shard2_replica_n4 s:shard2 c:collection1 r:core_node7) [n:127.0.0.1:42471_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_n4] o.a.s.c.SolrCore [collection1_shard2_replica_n4] Registered new searcher Searcher@176449d[collection1_shard2_replica_n4] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 790198 INFO  (qtp22959668-6853) [n:127.0.0.1:42471_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_n4] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1584396698432569344
   [junit4]   2> 790201 INFO  (qtp22959668-6790) [n:127.0.0.1:42471_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard1: total=2 found=1 timeoutin=9999ms
   [junit4]   2> 790202 INFO  (qtp22959668-6853) [n:127.0.0.1:42471_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_n4] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard2: total=2 found=1 timeoutin=9999ms
   [junit4]   2> 790253 INFO  (qtp13914217-6789) [n:127.0.0.1:45387_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_n2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 790253 INFO  (qtp13914217-6789) [n:127.0.0.1:45387_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_n2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 790254 INFO  (qtp13914217-6789) [n:127.0.0.1:45387_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_n2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 790254 INFO  (qtp13914217-6789) [n:127.0.0.1:45387_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_n2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 790256 INFO  (qtp13914217-6789) [n:127.0.0.1:45387_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@14f8ab7[collection1_shard1_replica_n2] main]
   [junit4]   2> 790257 INFO  (qtp13914217-6789) [n:127.0.0.1:45387_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_n2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/config
   [junit4]   2> 790259 INFO  (qtp13914217-6789) [n:127.0.0.1:45387_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_n2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/config
   [junit4]   2> 790260 INFO  (qtp13914217-6789) [n:127.0.0.1:45387_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_n2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 790260 INFO  (qtp13914217-6789) [n:127.0.0.1:45387_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_n2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1584396698497581056
   [junit4]   2> 790261 INFO  (searcherExecutor-3020-thread-1-processing-n:127.0.0.1:45387_solr x:collection1_shard1_replica_n2 s:shard1 c:collection1 r:core_node5) [n:127.0.0.1:45387_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_n2] o.a.s.c.SolrCore [collection1_shard1_replica_n2] Registered new searcher Searcher@14f8ab7[collection1_shard1_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 790265 INFO  (qtp13914217-6798) [n:127.0.0.1:45387_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_n6] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 790265 INFO  (qtp13914217-6798) [n:127.0.0.1:45387_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_n6] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 790266 INFO  (qtp13914217-6798) [n:127.0.0.1:45387_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_n6] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 790266 INFO  (qtp13914217-6798) [n:127.0.0.1:45387_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_n6] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 790268 INFO  (qtp13914217-6798) [n:127.0.0.1:45387_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_n6] o.a.s.s.SolrIndexSearcher Opening [Searcher@15a0f2c[collection1_shard2_replica_n6] main]
   [junit4]   2> 790270 INFO  (qtp13914217-6798) [n:127.0.0.1:45387_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_n6] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/config
   [junit4]   2> 790270 INFO  (qtp13914217-6798) [n:127.0.0.1:45387_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_n6] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/config
   [junit4]   2> 790271 INFO  (qtp13914217-6798) [n:127.0.0.1:45387_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_n6] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 790272 INFO  (searcherExecutor-3021-thread-1-processing-n:127.0.0.1:45387_solr x:collection1_shard2_replica_n6 s:shard2 c:collection1 r:core_node8) [n:127.0.0.1:45387_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_n6] o.a.s.c.SolrCore [collection1_shard2_replica_n6] Registered new searcher Searcher@15a0f2c[collection1_shard2_replica_n6] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 790272 INFO  (qtp13914217-6798) [n:127.0.0.1:45387_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_n6] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1584396698510163968
   [junit4]   2> 790303 INFO  (zkCallback-1307-thread-2-processing-n:127.0.0.1:45387_solr) [n:127.0.0.1:45387_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 790303 INFO  (zkCallback-1307-thread-3-processing-n:127.0.0.1:45387_solr) [n:127.0.0.1:45387_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 790303 INFO  (zkCallback-1308-thread-1-processing-n:127.0.0.1:42471_solr) [n:127.0.0.1:42471_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 790702 INFO  (qtp22959668-6790) [n:127.0.0.1:42471_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 790702 INFO  (qtp22959668-6790) [n:127.0.0.1:42471_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 790702 INFO  (qtp22959668-6790) [n:127.0.0.1:42471_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:42471/solr/collection1_shard1_replica_n1/
   [junit4]   2> 790702 INFO  (qtp22959668-6790) [n:127.0.0.1:42471_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.PeerSync PeerSync: core=collection1_shard1_replica_n1 url=https://127.0.0.1:42471/solr START replicas=[https://127.0.0.1:45387/solr/collection1_shard1_replica_n2/] nUpdates=100
   [junit4]   2> 790702 INFO  (qtp22959668-6853) [n:127.0.0.1:42471_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_n4] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 790702 INFO  (qtp22959668-6853) [n:127.0.0.1:42471_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_n4] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 790703 INFO  (qtp22959668-6853) [n:127.0.0.1:42471_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_n4] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:42471/solr/collection1_shard2_replica_n4/
   [junit4]   2> 790703 INFO  (qtp22959668-6853) [n:127.0.0.1:42471_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_n4] o.a.s.u.PeerSync PeerSync: core=collection1_shard2_replica_n4 url=https://127.0.0.1:42471/solr START replicas=[https://127.0.0.1:45387/solr/collection1_shard2_replica_n6/] nUpdates=100
   [junit4]   2> 790708 INFO  (qtp13914217-6872) [n:127.0.0.1:45387_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_n2] o.a.s.c.S.Request [collection1_shard1_replica_n2]  webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 790709 INFO  (qtp13914217-6794) [n:127.0.0.1:45387_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_n6] o.a.s.c.S.Request [collection1_shard2_replica_n6]  webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 791003 INFO  (qtp22959668-6790) [n:127.0.0.1:42471_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.PeerSync PeerSync: core=collection1_shard1_replica_n1 url=https://127.0.0.1:42471/solr DONE.  We have no versions.  sync failed.
   [junit4]   2> 791003 INFO  (qtp22959668-6790) [n:127.0.0.1:42471_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
   [junit4]   2> 791003 INFO  (qtp22959668-6790) [n:127.0.0.1:42471_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 791003 INFO  (qtp22959668-6790) [n:127.0.0.1:42471_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 791004 INFO  (qtp22959668-6853) [n:127.0.0.1:42471_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_n4] o.a.s.u.PeerSync PeerSync: core=collection1_shard2_replica_n4 url=https://127.0.0.1:42471/solr DONE.  We have no versions.  sync failed.
   [junit4]   2> 791004 INFO  (qtp22959668-6853) [n:127.0.0.1:42471_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_n4] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
   [junit4]   2> 791004 INFO  (qtp22959668-6853) [n:127.0.0.1:42471_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_n4] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 791004 INFO  (qtp22959668-6853) [n:127.0.0.1:42471_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_n4] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 791005 INFO  (qtp22959668-6790) [n:127.0.0.1:42471_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:42471/solr/collection1_shard1_replica_n1/ shard1
   [junit4]   2> 791006 INFO  (qtp22959668-6853) [n:127.0.0.1:42471_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_n4] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:42471/solr/collection1_shard2_replica_n4/ shard2
   [junit4]   2> 791107 INFO  (zkCallback-1307-thread-2-processing-n:127.0.0.1:45387_solr) [n:127.0.0.1:45387_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 791107 INFO  (zkCallback-1307-thread-3-processing-n:127.0.0.1:45387_solr) [n:127.0.0.1:45387_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 791107 INFO  (zkCallback-1308-thread-1-processing-n:127.0.0.1:42471_solr) [n:127.0.0.1:42471_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 791156 INFO  (qtp22959668-6790) [n:127.0.0.1:42471_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 791156 INFO  (qtp22959668-6853) [n:127.0.0.1:42471_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_n4] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 791164 INFO  (qtp22959668-6790) [n:127.0.0.1:42471_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=config&newCollection=true&name=collection1_shard1_replica_n1&action=CREATE&numShards=2&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2112
   [junit4]   2> 791164 INFO  (qtp22959668-6853) [n:127.0.0.1:42471_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_n4] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node7&collection.configName=config&newCollection=true&name=collection1_shard2_replica_n4&action=CREATE&numShards=2&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2112
   [junit4]   2> 791258 INFO  (zkCallback-1307-thread-3-processing-n:127.0.0.1:45387_solr) [n:127.0.0.1:45387_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 791258 INFO  (zkCallback-1307-thread-2-processing-n:127.0.0.1:45387_solr) [n:127.0.0.1:45387_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 791258 INFO  (zkCallback-1308-thread-1-processing-n:127.0.0.1:42471_solr) [n:127.0.0.1:42471_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 791265 INFO  (qtp13914217-6789) [n:127.0.0.1:45387_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_n2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node5&collection.configName=config&newCollection=true&name=collection1_shard1_replica_n2&action=CREATE&numShards=2&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2113
   [junit4]   2> 791276 INFO  (qtp13914217-6798) [n:127.0.0.1:45387_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_n6] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node8&collection.configName=config&newCollection=true&name=collection1_shard2_replica_n6&action=CREATE&numShards=2&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2127
   [junit4]   2> 791280 INFO  (qtp22959668-6788) [n:127.0.0.1:42471_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 791377 INFO  (zkCallback-1307-thread-2-processing-n:127.0.0.1:45387_solr) [n:127.0.0.1:45387_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 791377 INFO  (zkCallback-1307-thread-3-processing-n:127.0.0.1:45387_solr) [n:127.0.0.1:45387_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 791377 INFO  (zkCallback-1308-thread-1-processing-n:127.0.0.1:42471_solr) [n:127.0.0.1:42471_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 792280 INFO  (qtp22959668-6788) [n:127.0.0.1:42471_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={pullReplicas=0&replicationFactor=2&collection.configName=config&maxShardsPerNode=2&name=collection1&nrtReplicas=2&action=CREATE&numShards=2&tlogReplicas=0&wt=javabin&version=2} status=0 QTime=3550
   [junit4]   2> 792281 INFO  (SUITE-TestCloudRecovery-seed#[EEA86119200536F7]-worker) [    ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: collection1 failOnTimeout:true timeout (sec):30
   [junit4]   2> 792282 INFO  (SUITE-TestCloudRecovery-seed#[EEA86119200536F7]-worker) [    ] o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: collection1
   [junit4]   2> 792295 INFO  (TEST-TestCloudRecovery.leaderRecoverFromLogOnStartupTest-seed#[EEA86119200536F7]) [    ] o.a.s.SolrTestCaseJ4 ###Starting leaderRecoverFromLogOnStartupTest
   [junit4]   2> 792306 INFO  (qtp13914217-6873) [n:127.0.0.1:45387_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_n2]  webapp=/solr path=/update params={update.distrib=FROMLEADER&_version_=-1584396700633530368&distrib.from=https://127.0.0.1:42471/solr/collection1_shard1_replica_n1/&wt=javabin&version=2}{deleteByQuery=*:* (-1584396700633530368)} 0 3
   [junit4]   2> 792313 INFO  (qtp13914217-6871) [n:127.0.0.1:45387_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_n6] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard2_replica_n6]  webapp=/solr path=/update params={update.distrib=FROMLEADER&_version_=-1584396700642967552&distrib.from=https://127.0.0.1:42471/solr/collection1_shard2_replica_n4/&wt=javabin&version=2}{deleteByQuery=*:* (-1584396700642967552)} 0 2
   [junit4]   2> 792313 INFO  (qtp22959668-6788) [n:127.0.0.1:42471_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_n4] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard2_replica_n4]  webapp=/solr path=/update params={update.distrib=TOLEADER&distrib.from=https://127.0.0.1:42471/solr/collection1_shard1_replica_n1/&wt=javabin&version=2}{deleteByQuery=*:* (-1584396700642967552)} 0 8
   [junit4]   2> 792314 INFO  (qtp22959668-6797) [n:127.0.0.1:42471_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{deleteByQuery=*:* (-1584396700633530368)} 0 17
   [junit4]   2> 792316 INFO  (qtp22959668-6793) [n:127.0.0.1:42471_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1584396700653453312,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 792316 INFO  (qtp22959668-6793) [n:127.0.0.1:42471_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 792317 INFO  (qtp22959668-6793) [n:127.0.0.1:42471_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 792317 INFO  (qtp22959668-6793) [n:127.0.0.1:42471_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_n1]  webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=https://127.0.0.1:42471/solr/collection1_shard2_replica_n4/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 0
   [junit4]   2> 792317 INFO  (qtp13914217-6874) [n:127.0.0.1:45387_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_n2] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1584396700654501888,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 792317 INFO  (qtp13914217-6874) [n:127.0.0.1:45387_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_n2] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 792318 INFO  (qtp13914217-6874) [n:127.0.0.1:45387_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_n2] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 792318 INFO  (qtp13914217-6874) [n:127.0.0.1:45387_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_n2]  webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=https://127.0.0.1:42471/solr/collection1_shard2_replica_n4/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 1
   [junit4]   2> 792318 INFO  (qtp13914217-6792) [n:127.0.0.1:45387_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_n6] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1584396700655550464,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 792318 INFO  (qtp13914217-6792) [n:127.0.0.1:45387_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_n6] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 792319 INFO  (qtp13914217-6792) [n:127.0.0.1:45387_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_n6] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 792319 INFO  (qtp13914217-6792) [n:127.0.0.1:45387_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_n6] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard2_replica_n6]  webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=https://127.0.0.1:42471/solr/collection1_shard2_replica_n4/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 1
   [junit4]   2> 792320 INFO  (qtp22959668-6788) [n:127.0.0.1:42471_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_n4] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1584396700657647616,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 792320 INFO  (qtp22959668-6788) [n:127.0.0.1:42471_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_n4] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 792320 INFO  (qtp22959668-6788) [n:127.0.0.1:42471_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_n4] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 792320 INFO  (qtp22959668-6788) [n:127.0.0.1:42471_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_n4] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard2_replica_n4]  webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=https://127.0.0.1:42471/solr/collection1_shard2_replica_n4/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 0
   [junit4]   2> 792321 INFO  (qtp22959668-6799) [n:127.0.0.1:42471_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_n4] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard2_replica_n4]  webapp=/solr path=/update params={_stateVer_=collection1:6&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 5
   [junit4]   2> 792328 INFO  (qtp13914217-6794) [n:127.0.0.1:45387_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_n2]  webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:42471/solr/collection1_shard1_replica_n1/&wt=javabin&version=2}{add=[1 (15

[...truncated too long message...]

oud.ZkController.getLeader(ZkController.java:1178)
   [junit4]   2> 	... 41 more
   [junit4]   2> 
   [junit4]   2> 2160389 INFO  (qtp846579-11364) [n:127.0.0.1:41763_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=630253
   [junit4]   2> 2160391 ERROR (TEST-BasicDistributedZkTest.test-seed#[EEA86119200536F7]) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 2160392 INFO  (TEST-BasicDistributedZkTest.test-seed#[EEA86119200536F7]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:45645 45645
   [junit4]   2> 2166930 INFO  (Thread-3431) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:45645 45645
   [junit4]   2> 2166931 WARN  (Thread-3431) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	2	/solr/aliases.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	2	/solr/clusterstate.json
   [junit4]   2> 	2	/solr/clusterprops.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	2	/solr/live_nodes
   [junit4]   2> 	2	/solr/collections
   [junit4]   2> 
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=BasicDistributedZkTest -Dtests.method=test -Dtests.seed=EEA86119200536F7 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=es-PA -Dtests.timezone=Asia/Omsk -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR    638s J1 | BasicDistributedZkTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:41763
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([EEA86119200536F7:66FC5EC38EF95B0F]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:654)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1103)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:883)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:816)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:314)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:991)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > Caused by: java.net.SocketTimeoutException: Read timed out
   [junit4]    > 	at java.net.SocketInputStream.socketRead0(Native Method)
   [junit4]    > 	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
   [junit4]    > 	at java.net.SocketInputStream.read(SocketInputStream.java:171)
   [junit4]    > 	at java.net.SocketInputStream.read(SocketInputStream.java:141)
   [junit4]    > 	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
   [junit4]    > 	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
   [junit4]    > 	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
   [junit4]    > 	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
   [junit4]    > 	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
   [junit4]    > 	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
   [junit4]    > 	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
   [junit4]    > 	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
   [junit4]    > 	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
   [junit4]    > 	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:542)
   [junit4]    > 	... 43 more
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest_EEA86119200536F7-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=1872, maxMBSortInHeap=5.43357973547461, sim=RandomSimilarity(queryNorm=false): {}, locale=es-PA, timezone=Asia/Omsk
   [junit4]   2> NOTE: Linux 4.10.0-37-generic i386/Oracle Corporation 1.8.0_144 (32-bit)/cpus=8,threads=1,free=174398032,total=518979584
   [junit4]   2> NOTE: All tests run in this JVM: [QueryEqualityTest, TestManagedSynonymFilterFactory, MetricsConfigTest, TestConfigSetsAPIExclusivity, TestDynamicLoading, TestAnalyzedSuggestions, ZkSolrClientTest, TestConfigReload, MoveReplicaHDFSFailoverTest, TestStressRecovery, TestSolr4Spatial2, CreateCollectionCleanupTest, PeerSyncWithIndexFingerprintCachingTest, TestFaceting, PropertiesRequestHandlerTest, TestWriterPerf, TestAtomicUpdateErrorCases, NotRequiredUniqueKeyTest, TestUseDocValuesAsStored, TestRandomFlRTGCloud, TestLockTree, TestCursorMarkWithoutUniqueKey, TestDocumentBuilder, DistributedExpandComponentTest, RuleEngineTest, TestCollationField, TestOverriddenPrefixQueryForCustomFieldType, TestLegacyFieldReuse, TestReplicaProperties, SolrIndexConfigTest, CoreMergeIndexesAdminHandlerTest, TestSizeLimitedDistributedMap, TestStressInPlaceUpdates, TestManagedSchema, SolrCLIZkUtilsTest, AddSchemaFieldsUpdateProcessorFactoryTest, SOLR749Test, LegacyCloudClusterPropTest, TestRemoteStreaming, AnalyticsQueryTest, DeleteShardTest, UtilsToolTest, TestPerFieldSimilarity, TestScoreJoinQPScore, DistributedIntervalFacetingTest, SpatialRPTFieldTypeTest, ShowFileRequestHandlerTest, PathHierarchyTokenizerFactoryTest, TestZkAclsWithHadoopAuth, TestHighFrequencyDictionaryFactory, RegexBytesRefFilterTest, ClassificationUpdateProcessorFactoryTest, TestUnifiedSolrHighlighter, SolrSlf4jReporterTest, TestSystemCollAutoCreate, DateRangeFieldTest, ParsingFieldUpdateProcessorsTest, TestNumericTerms64, TestXmlQParser, ShardRoutingCustomTest, MoreLikeThisHandlerTest, DocumentAnalysisRequestHandlerTest, SuggesterTSTTest, SchemaVersionSpecificBehaviorTest, TestSimpleTrackingShardHandler, HdfsWriteToMultipleCollectionsTest, TestFieldCacheSort, FullHLLTest, SolrMetricReporterTest, BooleanFieldTest, TestConfigsApi, NumericFieldsTest, PeerSyncReplicationTest, BadIndexSchemaTest, TestExpandComponent, HighlighterConfigTest, OutputWriterTest, TemplateUpdateProcessorTest, RollingRestartTest, TestTrackingShardHandlerFactory, TestPrepRecovery, CloudExitableDirectoryReaderTest, TestCloudRecovery, DistributedSuggestComponentTest, TestSchemaResource, TestMultiValuedNumericRangeQuery, TestBulkSchemaAPI, HdfsRecoveryZkTest, BinaryUpdateRequestHandlerTest, TestStandardQParsers, DirectUpdateHandlerTest, TestSortByMinMaxFunction, TestSerializedLuceneMatchVersion, TestFilteredDocIdSet, RegexBoostProcessorTest, TriggerIntegrationTest, TestDocSet, CdcrRequestHandlerTest, DebugComponentTest, HdfsCollectionsAPIDistributedZkTest, TestCloudNestedDocsSort, TestInPlaceUpdatesStandalone, NumberUtilsTest, StatsReloadRaceTest, MultiThreadedOCPTest, TestComplexPhraseQParserPlugin, SolrGraphiteReporterTest, CollectionsAPIDistributedZkTest, SolrRequestParserTest, DistributedFacetPivotLongTailTest, DeleteLastCustomShardedReplicaTest, TestQuerySenderNoQuery, TestStressReorder, TestSubQueryTransformer, TestManagedResource, ZkCLITest, LukeRequestHandlerTest, TestExclusionRuleCollectionAccess, MergeStrategyTest, TestFieldTypeCollectionResource, TestReRankQParserPlugin, BigEndianAscendingWordSerializerTest, TestIntervalFaceting, TestExceedMaxTermLength, BasicDistributedZkTest]
   [junit4] Completed [358/749 (5!)] on J1 in 637.55s, 1 test, 1 error <<< FAILURES!

[...truncated 44477 lines...]

[JENKINS] Lucene-Solr-7.x-Linux (32bit/jdk1.8.0_144) - Build # 846 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Linux/846/
Java: 32bit/jdk1.8.0_144 -client -XX:+UseParallelGC

5 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.analytics.legacy.LegacyNoFacetCloudTest

Error Message:
Error from server at https://127.0.0.1:36913/solr/collection1: Async exception during distributed update: 127.0.0.1:34041 failed to respond

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:36913/solr/collection1: Async exception during distributed update: 127.0.0.1:34041 failed to respond
	at __randomizedtesting.SeedInfo.seed([7E0E823B4D219F81]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1103)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:883)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:816)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.request.UpdateRequest.commit(UpdateRequest.java:233)
	at org.apache.solr.analytics.legacy.LegacyAbstractAnalyticsCloudTest.cleanIndex(LegacyAbstractAnalyticsCloudTest.java:59)
	at org.apache.solr.analytics.legacy.LegacyAbstractAnalyticsCloudTest.setupCollection(LegacyAbstractAnalyticsCloudTest.java:53)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:874)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.update.processor.TestNamedUpdateProcessors

Error Message:
41 threads leaked from SUITE scope at org.apache.solr.update.processor.TestNamedUpdateProcessors:     1) Thread[id=256, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=284, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=243, name=qtp24271884-243, state=RUNNABLE, group=TGRP-TestNamedUpdateProcessors]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=315, name=zkCallback-64-thread-4, state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=316, name=zkCallback-64-thread-5, state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=312, name=zkCallback-58-thread-5, state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    7) Thread[id=285, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)    8) Thread[id=279, name=qtp20321829-279, state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)    9) Thread[id=248, name=qtp24271884-248, state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   10) Thread[id=309, name=zkCallback-58-thread-3-processing-n:127.0.0.1:44065_, state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.create(SolrZkClient.java:357)         at org.apache.solr.cloud.Overseer.createOverseerNode(Overseer.java:771)         at org.apache.solr.cloud.Overseer.getStateUpdateQueue(Overseer.java:644)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.<init>(Overseer.java:107)         at org.apache.solr.cloud.Overseer.start(Overseer.java:518)         at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:799)         at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)         at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)         at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:56)         at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:348)         at org.apache.solr.common.cloud.SolrZkClient$1.lambda$process$0(SolrZkClient.java:262)         at org.apache.solr.common.cloud.SolrZkClient$1$$Lambda$8/31224210.run(Unknown Source)         at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)         at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$9/31908684.run(Unknown Source)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   11) Thread[id=278, name=qtp20321829-278, state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.ZkController.waitForShardId(ZkController.java:1561)         at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1512)         at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1621)         at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1030)         at org.apache.solr.core.CoreContainer.create(CoreContainer.java:948)         at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)         at org.apache.solr.handler.admin.CoreAdminOperation$$Lambda$191/22304042.execute(Unknown Source)         at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)         at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)         at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)         at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)         at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:735)         at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:716)         at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:497)         at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)         at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)         at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)         at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)         at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)         at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)         at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)         at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)         at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)         at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)         at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)         at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)         at org.eclipse.jetty.server.Server.handle(Server.java:534)         at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)         at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)         at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)         at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)         at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:251)         at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)         at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)         at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:748)   12) Thread[id=294, name=Scheduler-11441993, state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   13) Thread[id=304, name=qtp20321829-304, state=RUNNABLE, group=TGRP-TestNamedUpdateProcessors]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:748)   14) Thread[id=246, name=qtp24271884-246, state=RUNNABLE, group=TGRP-TestNamedUpdateProcessors]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:748)   15) Thread[id=269, name=searcherExecutor-117-thread-1, state=WAITING, group=TGRP-TestNamedUpdateProcessors]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   16) Thread[id=273, name=qtp20321829-273, state=RUNNABLE, group=TGRP-TestNamedUpdateProcessors]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:748)   17) Thread[id=277, name=qtp20321829-277, state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   18) Thread[id=265, name=Scheduler-6664168, state=WAITING, group=TGRP-TestNamedUpdateProcessors]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   19) Thread[id=275, name=qtp20321829-275, state=RUNNABLE, group=TGRP-TestNamedUpdateProcessors]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:748)   20) Thread[id=251, name=org.eclipse.jetty.server.session.HashSessionManager@1eac0aTimer, state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   21) Thread[id=280, name=org.eclipse.jetty.server.session.HashSessionManager@80e680Timer, state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   22) Thread[id=244, name=qtp24271884-244, state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   23) Thread[id=314, name=zkCallback-58-thread-7, state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   24) Thread[id=313, name=zkCallback-58-thread-6, state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   25) Thread[id=250, name=qtp24271884-250, state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   26) Thread[id=245, name=qtp24271884-245, state=RUNNABLE, group=TGRP-TestNamedUpdateProcessors]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:748)   27) Thread[id=272, name=qtp20321829-272, state=RUNNABLE, group=TGRP-TestNamedUpdateProcessors]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:748)   28) Thread[id=263, name=Thread-53, state=WAITING, group=TGRP-TestNamedUpdateProcessors]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1717)   29) Thread[id=276, name=qtp20321829-276-acceptor-0@1c4efc4-ServerConnector@303d2b{SSL,[ssl, http/1.1]}{127.0.0.1:42733}, state=RUNNABLE, group=TGRP-TestNamedUpdateProcessors]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:748)   30) Thread[id=292, name=Thread-62, state=WAITING, group=TGRP-TestNamedUpdateProcessors]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1717)   31) Thread[id=249, name=qtp24271884-249, state=RUNNABLE, group=TGRP-TestNamedUpdateProcessors]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:748)   32) Thread[id=290, name=TEST-TestNamedUpdateProcessors.test-seed#[BDCB89DD509B725A]-EventThread, state=WAITING, group=TGRP-TestNamedUpdateProcessors]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)   33) Thread[id=311, name=zkCallback-58-thread-4, state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   34) Thread[id=260, name=TEST-TestNamedUpdateProcessors.test-seed#[BDCB89DD509B725A]-SendThread(127.0.0.1:35347), state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)   35) Thread[id=289, name=TEST-TestNamedUpdateProcessors.test-seed#[BDCB89DD509B725A]-SendThread(127.0.0.1:35347), state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)   36) Thread[id=261, name=TEST-TestNamedUpdateProcessors.test-seed#[BDCB89DD509B725A]-EventThread, state=WAITING, group=TGRP-TestNamedUpdateProcessors]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)   37) Thread[id=271, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)   38) Thread[id=317, name=zkCallback-64-thread-6, state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   39) Thread[id=247, name=qtp24271884-247-acceptor-0@190d79f-ServerConnector@4acc36{SSL,[ssl, http/1.1]}{127.0.0.1:44065}, state=RUNNABLE, group=TGRP-TestNamedUpdateProcessors]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:748)   40) Thread[id=255, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)   41) Thread[id=318, name=zkCallback-64-thread-7, state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 41 threads leaked from SUITE scope at org.apache.solr.update.processor.TestNamedUpdateProcessors: 
   1) Thread[id=256, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=284, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=243, name=qtp24271884-243, state=RUNNABLE, group=TGRP-TestNamedUpdateProcessors]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=315, name=zkCallback-64-thread-4, state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=316, name=zkCallback-64-thread-5, state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=312, name=zkCallback-58-thread-5, state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   7) Thread[id=285, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
   8) Thread[id=279, name=qtp20321829-279, state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
   9) Thread[id=248, name=qtp24271884-248, state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  10) Thread[id=309, name=zkCallback-58-thread-3-processing-n:127.0.0.1:44065_, state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.create(SolrZkClient.java:357)
        at org.apache.solr.cloud.Overseer.createOverseerNode(Overseer.java:771)
        at org.apache.solr.cloud.Overseer.getStateUpdateQueue(Overseer.java:644)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.<init>(Overseer.java:107)
        at org.apache.solr.cloud.Overseer.start(Overseer.java:518)
        at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:799)
        at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)
        at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)
        at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:56)
        at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:348)
        at org.apache.solr.common.cloud.SolrZkClient$1.lambda$process$0(SolrZkClient.java:262)
        at org.apache.solr.common.cloud.SolrZkClient$1$$Lambda$8/31224210.run(Unknown Source)
        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$9/31908684.run(Unknown Source)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  11) Thread[id=278, name=qtp20321829-278, state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.ZkController.waitForShardId(ZkController.java:1561)
        at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1512)
        at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1621)
        at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1030)
        at org.apache.solr.core.CoreContainer.create(CoreContainer.java:948)
        at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
        at org.apache.solr.handler.admin.CoreAdminOperation$$Lambda$191/22304042.execute(Unknown Source)
        at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
        at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
        at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
        at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
        at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:735)
        at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:716)
        at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:497)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
        at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
        at org.eclipse.jetty.server.Server.handle(Server.java:534)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
        at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:251)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
        at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:748)
  12) Thread[id=294, name=Scheduler-11441993, state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  13) Thread[id=304, name=qtp20321829-304, state=RUNNABLE, group=TGRP-TestNamedUpdateProcessors]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:748)
  14) Thread[id=246, name=qtp24271884-246, state=RUNNABLE, group=TGRP-TestNamedUpdateProcessors]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
        at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:748)
  15) Thread[id=269, name=searcherExecutor-117-thread-1, state=WAITING, group=TGRP-TestNamedUpdateProcessors]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  16) Thread[id=273, name=qtp20321829-273, state=RUNNABLE, group=TGRP-TestNamedUpdateProcessors]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:748)
  17) Thread[id=277, name=qtp20321829-277, state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  18) Thread[id=265, name=Scheduler-6664168, state=WAITING, group=TGRP-TestNamedUpdateProcessors]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  19) Thread[id=275, name=qtp20321829-275, state=RUNNABLE, group=TGRP-TestNamedUpdateProcessors]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
        at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:748)
  20) Thread[id=251, name=org.eclipse.jetty.server.session.HashSessionManager@1eac0aTimer, state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  21) Thread[id=280, name=org.eclipse.jetty.server.session.HashSessionManager@80e680Timer, state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  22) Thread[id=244, name=qtp24271884-244, state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  23) Thread[id=314, name=zkCallback-58-thread-7, state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  24) Thread[id=313, name=zkCallback-58-thread-6, state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  25) Thread[id=250, name=qtp24271884-250, state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  26) Thread[id=245, name=qtp24271884-245, state=RUNNABLE, group=TGRP-TestNamedUpdateProcessors]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:748)
  27) Thread[id=272, name=qtp20321829-272, state=RUNNABLE, group=TGRP-TestNamedUpdateProcessors]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:748)
  28) Thread[id=263, name=Thread-53, state=WAITING, group=TGRP-TestNamedUpdateProcessors]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1717)
  29) Thread[id=276, name=qtp20321829-276-acceptor-0@1c4efc4-ServerConnector@303d2b{SSL,[ssl, http/1.1]}{127.0.0.1:42733}, state=RUNNABLE, group=TGRP-TestNamedUpdateProcessors]
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
        at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371)
        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:748)
  30) Thread[id=292, name=Thread-62, state=WAITING, group=TGRP-TestNamedUpdateProcessors]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1717)
  31) Thread[id=249, name=qtp24271884-249, state=RUNNABLE, group=TGRP-TestNamedUpdateProcessors]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:748)
  32) Thread[id=290, name=TEST-TestNamedUpdateProcessors.test-seed#[BDCB89DD509B725A]-EventThread, state=WAITING, group=TGRP-TestNamedUpdateProcessors]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
  33) Thread[id=311, name=zkCallback-58-thread-4, state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  34) Thread[id=260, name=TEST-TestNamedUpdateProcessors.test-seed#[BDCB89DD509B725A]-SendThread(127.0.0.1:35347), state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
  35) Thread[id=289, name=TEST-TestNamedUpdateProcessors.test-seed#[BDCB89DD509B725A]-SendThread(127.0.0.1:35347), state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
  36) Thread[id=261, name=TEST-TestNamedUpdateProcessors.test-seed#[BDCB89DD509B725A]-EventThread, state=WAITING, group=TGRP-TestNamedUpdateProcessors]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
  37) Thread[id=271, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
  38) Thread[id=317, name=zkCallback-64-thread-6, state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  39) Thread[id=247, name=qtp24271884-247-acceptor-0@190d79f-ServerConnector@4acc36{SSL,[ssl, http/1.1]}{127.0.0.1:44065}, state=RUNNABLE, group=TGRP-TestNamedUpdateProcessors]
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
        at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371)
        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:748)
  40) Thread[id=255, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
  41) Thread[id=318, name=zkCallback-64-thread-7, state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([BDCB89DD509B725A]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.update.processor.TestNamedUpdateProcessors

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=243, name=qtp24271884-243, state=RUNNABLE, group=TGRP-TestNamedUpdateProcessors]         at java.lang.Thread.isInterrupted(Native Method)         at java.lang.Thread.isInterrupted(Thread.java:964)         at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:218)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=315, name=zkCallback-64-thread-4, state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=316, name=zkCallback-64-thread-5, state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=312, name=zkCallback-58-thread-5, state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=309, name=zkCallback-58-thread-3, state=TIMED_WAITING, group=TGRP-TestNamedUpdateProcessors]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=294, name=Scheduler-11441993, state=WAITING, group=TGRP-TestNamedUpdateProcessors]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    7) Thread[id=304, name=qtp20321829-304, state=RUNNABLE, group=TGRP-TestNamedUpdateProcessors]         at sun.nio.ch.IOUtil.drain(Native Method)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:748)    8) Thread[id=246, name=qtp24271884-246, state=RUNNABLE, group=TGRP-TestNamedUpdateProcessors]         at sun.nio.ch.IOUtil.drain(Native Method)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:748)    9) Thread[id=269, name=searcherExecutor-117-thread-1, state=WAITING, group=TGRP-TestNamedUpdateProcessors]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   10) Thread[id=273, name=qtp20321829-273, state=RUNNABLE, group=TGRP-TestNamedUpdateProcessors]         at java.lang.Thread.isInterrupted(Native Method)         at java.lang.Thread.isInterrupted(Thread.java:964)         at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:218)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)         at org.eclipse.jetty.util.thread.QueuedT

[...truncated too long message...]

ctChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: No registered leader was found after waiting for 1220000ms , collection: collection1 slice: shard1 saw state=null with live_nodes=[]
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:771)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:736)
   [junit4]   2> 	at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1178)
   [junit4]   2> 	... 41 more
   [junit4]   2> 
   [junit4]   2> 319925 INFO  (qtp455548-1428) [n:127.0.0.1:41099_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=collection1_shard1_replica_n1&action=CREATE&numShards=2&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=209789
   [junit4]   2> 319926 ERROR (SUITE-GraphExpressionTest-seed#[F07D39A5103EECBE]-worker) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 319926 INFO  (SUITE-GraphExpressionTest-seed#[F07D39A5103EECBE]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:40189 40189
   [junit4]   2> 322000 INFO  (Thread-311) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:40189 40189
   [junit4]   2> 322001 WARN  (Thread-311) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	3	/solr/aliases.json
   [junit4]   2> 	3	/solr/clusterprops.json
   [junit4]   2> 	2	/solr/security.json
   [junit4]   2> 	2	/solr/configs/conf
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	6	/solr/collections/collection1/state.json
   [junit4]   2> 	3	/solr/clusterstate.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	3	/solr/live_nodes
   [junit4]   2> 	3	/solr/collections
   [junit4]   2> 
   [junit4]   2> 322001 INFO  (SUITE-GraphExpressionTest-seed#[F07D39A5103EECBE]-worker) [    ] o.a.s.SolrTestCaseJ4 ------------------------------------------------------- Done waiting for tracked resources to be released
   [junit4]   2> NOTE: test params are: codec=FastCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST, chunkSize=9, maxDocsPerChunk=960, blockSize=596), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST, chunkSize=9, blockSize=596)), sim=RandomSimilarity(queryNorm=false): {}, locale=pt-PT, timezone=Pacific/Gambier
   [junit4]   2> NOTE: Linux 4.10.0-37-generic i386/Oracle Corporation 1.8.0_144 (32-bit)/cpus=8,threads=1,free=125744192,total=253493248
   [junit4]   2> NOTE: All tests run in this JVM: [SolrExampleStreamingBinaryTest, GraphTest, TestDelegationTokenRequest, SolrExampleEmbeddedTest, OrEvaluatorTest, RecursiveEvaluatorTest, SolrZkClientTest, TestFastInputStream, SolrQueryTest, MergeIndexesEmbeddedTest, ConcatOperationTest, CorrelationEvaluatorTest, NormalDistributionEvaluatorTest, UniformDistributionEvaluatorTest, ConcurrentUpdateSolrClientTest, TestUpdateRequestCodec, HyperbolicCosineEvaluatorTest, CommonAdminParamsTest, TestSolrProperties, CumulativeProbabilityEvaluatorTest, SolrPingTest, ArcTangentEvaluatorTest, CeilingEvaluatorTest, SolrExampleBinaryTest, SolrExampleXMLTest, LargeVolumeBinaryJettyTest, JettyWebappTest, TestSpellCheckResponse, SolrParamTest, TermsResponseTest, QueryResponseTest, TestCoreAdmin, TestDocumentObjectBinder, ModifiableSolrParamsTest, SolrDocumentTest, ClientUtilsTest, FacetFieldTest, UsingSolrJRefGuideExamplesTest, CollectionAdminRequestRequiredParamsTest, GetByIdTest, TestLBHttpSolrClient, TestPolicy, BasicHttpSolrClientTest, CloudSolrClientMultiConstructorTest, ConcurrentUpdateSolrClientBuilderTest, HttpSolrClientBuilderTest, HttpSolrClientConPoolTest, HttpSolrClientSSLAuthConPoolTest, GraphExpressionTest]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=GraphExpressionTest -Dtests.seed=F07D39A5103EECBE -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=pt-PT -Dtests.timezone=Pacific/Gambier -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   0.00s J1 | GraphExpressionTest (suite) <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:33277/solr: create the collection time out:180s
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([F07D39A5103EECBE]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1103)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:883)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:816)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
   [junit4]    > 	at org.apache.solr.client.solrj.io.graph.GraphExpressionTest.setupCluster(GraphExpressionTest.java:87)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4] Completed [146/146 (1!)] on J1 in 212.68s, 0 tests, 1 error <<< FAILURES!

[...truncated 2005 lines...]
   [junit4] Suite: org.apache.solr.analytics.legacy.LegacyNoFacetCloudTest
   [junit4]   2> Creating dataDir: /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/contrib/solr-analytics/test/J2/temp/solr.analytics.legacy.LegacyNoFacetCloudTest_7E0E823B4D219F81-001/init-core-data-001
   [junit4]   2> Nov 18, 2017 7:07:32 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarity(queryNorm=false): {}, locale=de-LU, timezone=Africa/Harare
   [junit4]   2> NOTE: Linux 4.10.0-37-generic i386/Oracle Corporation 1.8.0_144 (32-bit)/cpus=8,threads=1,free=93258160,total=208142336
   [junit4]   2> NOTE: All tests run in this JVM: [OverallAnalyticsTest, LegacyFieldFacetExtrasTest, FillMissingFunctionTest, ConstantValueTest, CastingDoubleValueTest, CastingIntValueStreamTest, TopFunctionTest, PowerFunctionTest, DivideFunctionTest, LegacyNoFacetCloudTest]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LegacyNoFacetCloudTest -Dtests.seed=7E0E823B4D219F81 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=de-LU -Dtests.timezone=Africa/Harare -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   0.00s J2 | LegacyNoFacetCloudTest (suite) <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:36913/solr/collection1: Async exception during distributed update: 127.0.0.1:34041 failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([7E0E823B4D219F81]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1103)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:883)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:816)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
   [junit4]    > 	at org.apache.solr.client.solrj.request.UpdateRequest.commit(UpdateRequest.java:233)
   [junit4]    > 	at org.apache.solr.analytics.legacy.LegacyAbstractAnalyticsCloudTest.cleanIndex(LegacyAbstractAnalyticsCloudTest.java:59)
   [junit4]    > 	at org.apache.solr.analytics.legacy.LegacyAbstractAnalyticsCloudTest.setupCollection(LegacyAbstractAnalyticsCloudTest.java:53)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4] Completed [24/78 (1!)] on J2 in 6.53s, 0 tests, 1 error <<< FAILURES!

[...truncated 40234 lines...]