You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2017/07/28 00:54:05 UTC

[JENKINS] Lucene-Solr-7.x-MacOSX (64bit/jdk1.8.0) - Build # 79 - Still Failing!

Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-MacOSX/79/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseSerialGC

1 tests failed.
FAILED:  org.apache.solr.cloud.ReplaceNodeTest.test

Error Message:
Could not find collection : replacenodetest_coll

Stack Trace:
org.apache.solr.common.SolrException: Could not find collection : replacenodetest_coll
	at __randomizedtesting.SeedInfo.seed([20D3DA769E1BC5EF:A887E5AC30E7A817]:0)
	at org.apache.solr.common.cloud.ClusterState.getCollection(ClusterState.java:194)
	at org.apache.solr.cloud.ReplaceNodeTest.test(ReplaceNodeTest.java:123)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 11585 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplaceNodeTest
   [junit4]   2> 678546 INFO  (SUITE-ReplaceNodeTest-seed#[20D3DA769E1BC5EF]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_20D3DA769E1BC5EF-001/init-core-data-001
   [junit4]   2> 678547 WARN  (SUITE-ReplaceNodeTest-seed#[20D3DA769E1BC5EF]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
   [junit4]   2> 678547 INFO  (SUITE-ReplaceNodeTest-seed#[20D3DA769E1BC5EF]-worker) [    ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 678550 INFO  (SUITE-ReplaceNodeTest-seed#[20D3DA769E1BC5EF]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth
   [junit4]   2> 678550 INFO  (SUITE-ReplaceNodeTest-seed#[20D3DA769E1BC5EF]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 6 servers in /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_20D3DA769E1BC5EF-001/tempDir-001
   [junit4]   2> 678550 INFO  (SUITE-ReplaceNodeTest-seed#[20D3DA769E1BC5EF]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 678551 INFO  (Thread-2200) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 678551 INFO  (Thread-2200) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 678554 ERROR (Thread-2200) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 678660 INFO  (SUITE-ReplaceNodeTest-seed#[20D3DA769E1BC5EF]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:50287
   [junit4]   2> 678691 INFO  (jetty-launcher-1617-thread-3) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 678691 INFO  (jetty-launcher-1617-thread-2) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 678692 INFO  (jetty-launcher-1617-thread-4) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 678692 INFO  (jetty-launcher-1617-thread-1) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 678692 INFO  (jetty-launcher-1617-thread-5) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 678693 INFO  (jetty-launcher-1617-thread-6) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 678695 INFO  (jetty-launcher-1617-thread-3) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5330c51e{/solr,null,AVAILABLE}
   [junit4]   2> 678695 INFO  (jetty-launcher-1617-thread-6) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1351162f{/solr,null,AVAILABLE}
   [junit4]   2> 678696 INFO  (jetty-launcher-1617-thread-6) [    ] o.e.j.s.AbstractConnector Started ServerConnector@1b398175{HTTP/1.1,[http/1.1]}{127.0.0.1:50290}
   [junit4]   2> 678696 INFO  (jetty-launcher-1617-thread-3) [    ] o.e.j.s.AbstractConnector Started ServerConnector@54cdbc{HTTP/1.1,[http/1.1]}{127.0.0.1:50289}
   [junit4]   2> 678696 INFO  (jetty-launcher-1617-thread-6) [    ] o.e.j.s.Server Started @682105ms
   [junit4]   2> 678696 INFO  (jetty-launcher-1617-thread-3) [    ] o.e.j.s.Server Started @682105ms
   [junit4]   2> 678696 INFO  (jetty-launcher-1617-thread-6) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=50290}
   [junit4]   2> 678696 INFO  (jetty-launcher-1617-thread-3) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=50289}
   [junit4]   2> 678696 ERROR (jetty-launcher-1617-thread-6) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 678696 ERROR (jetty-launcher-1617-thread-3) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 678696 INFO  (jetty-launcher-1617-thread-6) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.1.0
   [junit4]   2> 678696 INFO  (jetty-launcher-1617-thread-6) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 678696 INFO  (jetty-launcher-1617-thread-6) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 678696 INFO  (jetty-launcher-1617-thread-6) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-07-27T23:55:14.551Z
   [junit4]   2> 678697 INFO  (jetty-launcher-1617-thread-3) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.1.0
   [junit4]   2> 678697 INFO  (jetty-launcher-1617-thread-3) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 678697 INFO  (jetty-launcher-1617-thread-3) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 678697 INFO  (jetty-launcher-1617-thread-3) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-07-27T23:55:14.552Z
   [junit4]   2> 678701 INFO  (jetty-launcher-1617-thread-4) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@54452cfd{/solr,null,AVAILABLE}
   [junit4]   2> 678702 INFO  (jetty-launcher-1617-thread-5) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5e7cdf9f{/solr,null,AVAILABLE}
   [junit4]   2> 678702 INFO  (jetty-launcher-1617-thread-3) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 678703 INFO  (jetty-launcher-1617-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@704b31c5{/solr,null,AVAILABLE}
   [junit4]   2> 678703 INFO  (jetty-launcher-1617-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@7bee1fd0{HTTP/1.1,[http/1.1]}{127.0.0.1:50295}
   [junit4]   2> 678704 INFO  (jetty-launcher-1617-thread-1) [    ] o.e.j.s.Server Started @682113ms
   [junit4]   2> 678704 INFO  (jetty-launcher-1617-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=50295}
   [junit4]   2> 678704 ERROR (jetty-launcher-1617-thread-1) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 678704 INFO  (jetty-launcher-1617-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.1.0
   [junit4]   2> 678704 INFO  (jetty-launcher-1617-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 678704 INFO  (jetty-launcher-1617-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 678704 INFO  (jetty-launcher-1617-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-07-27T23:55:14.559Z
   [junit4]   2> 678705 INFO  (jetty-launcher-1617-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@19b3dcef{/solr,null,AVAILABLE}
   [junit4]   2> 678706 INFO  (jetty-launcher-1617-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@7d746145{HTTP/1.1,[http/1.1]}{127.0.0.1:50296}
   [junit4]   2> 678706 INFO  (jetty-launcher-1617-thread-2) [    ] o.e.j.s.Server Started @682115ms
   [junit4]   2> 678706 INFO  (jetty-launcher-1617-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=50296}
   [junit4]   2> 678706 ERROR (jetty-launcher-1617-thread-2) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 678706 INFO  (jetty-launcher-1617-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.1.0
   [junit4]   2> 678706 INFO  (jetty-launcher-1617-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 678706 INFO  (jetty-launcher-1617-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 678706 INFO  (jetty-launcher-1617-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-07-27T23:55:14.561Z
   [junit4]   2> 678707 INFO  (jetty-launcher-1617-thread-4) [    ] o.e.j.s.AbstractConnector Started ServerConnector@6c8f11b3{HTTP/1.1,[http/1.1]}{127.0.0.1:50293}
   [junit4]   2> 678707 INFO  (jetty-launcher-1617-thread-4) [    ] o.e.j.s.Server Started @682116ms
   [junit4]   2> 678707 INFO  (jetty-launcher-1617-thread-4) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=50293}
   [junit4]   2> 678707 ERROR (jetty-launcher-1617-thread-4) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 678707 INFO  (jetty-launcher-1617-thread-4) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.1.0
   [junit4]   2> 678707 INFO  (jetty-launcher-1617-thread-4) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 678707 INFO  (jetty-launcher-1617-thread-4) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 678707 INFO  (jetty-launcher-1617-thread-4) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-07-27T23:55:14.562Z
   [junit4]   2> 678708 INFO  (jetty-launcher-1617-thread-6) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 678709 INFO  (jetty-launcher-1617-thread-5) [    ] o.e.j.s.AbstractConnector Started ServerConnector@59e36bc6{HTTP/1.1,[http/1.1]}{127.0.0.1:50294}
   [junit4]   2> 678709 INFO  (jetty-launcher-1617-thread-5) [    ] o.e.j.s.Server Started @682118ms
   [junit4]   2> 678709 INFO  (jetty-launcher-1617-thread-5) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=50294}
   [junit4]   2> 678709 ERROR (jetty-launcher-1617-thread-5) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 678709 INFO  (jetty-launcher-1617-thread-5) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.1.0
   [junit4]   2> 678709 INFO  (jetty-launcher-1617-thread-5) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 678709 INFO  (jetty-launcher-1617-thread-5) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 678709 INFO  (jetty-launcher-1617-thread-5) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-07-27T23:55:14.564Z
   [junit4]   2> 678711 INFO  (jetty-launcher-1617-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 678712 INFO  (jetty-launcher-1617-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 678715 INFO  (jetty-launcher-1617-thread-4) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 678717 INFO  (jetty-launcher-1617-thread-3) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:50287/solr
   [junit4]   2> 678726 INFO  (jetty-launcher-1617-thread-5) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 678729 INFO  (jetty-launcher-1617-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:50287/solr
   [junit4]   2> 678730 INFO  (jetty-launcher-1617-thread-6) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:50287/solr
   [junit4]   2> 678733 INFO  (jetty-launcher-1617-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:50287/solr
   [junit4]   2> 678735 INFO  (jetty-launcher-1617-thread-5) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:50287/solr
   [junit4]   2> 678735 INFO  (jetty-launcher-1617-thread-4) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:50287/solr
   [junit4]   2> 678951 INFO  (jetty-launcher-1617-thread-6) [n:127.0.0.1:50290_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 678951 INFO  (jetty-launcher-1617-thread-2) [n:127.0.0.1:50296_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 678951 INFO  (jetty-launcher-1617-thread-1) [n:127.0.0.1:50295_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 678951 INFO  (jetty-launcher-1617-thread-5) [n:127.0.0.1:50294_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 678952 INFO  (jetty-launcher-1617-thread-6) [n:127.0.0.1:50290_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:50290_solr
   [junit4]   2> 678954 INFO  (jetty-launcher-1617-thread-1) [n:127.0.0.1:50295_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50295_solr
   [junit4]   2> 678955 INFO  (jetty-launcher-1617-thread-2) [n:127.0.0.1:50296_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50296_solr
   [junit4]   2> 678956 INFO  (jetty-launcher-1617-thread-5) [n:127.0.0.1:50294_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50294_solr
   [junit4]   2> 678956 INFO  (jetty-launcher-1617-thread-6) [n:127.0.0.1:50290_solr    ] o.a.s.c.Overseer Overseer (id=98382624483442702-127.0.0.1:50290_solr-n_0000000000) starting
   [junit4]   2> 678958 INFO  (zkCallback-1648-thread-1-processing-n:127.0.0.1:50295_solr) [n:127.0.0.1:50295_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 678958 INFO  (zkCallback-1646-thread-1-processing-n:127.0.0.1:50296_solr) [n:127.0.0.1:50296_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 678959 INFO  (zkCallback-1639-thread-1-processing-n:127.0.0.1:50289_solr) [n:127.0.0.1:50289_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 678959 INFO  (zkCallback-1645-thread-1-processing-n:127.0.0.1:50290_solr) [n:127.0.0.1:50290_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 678959 INFO  (zkCallback-1649-thread-1-processing-n:127.0.0.1:50293_solr) [n:127.0.0.1:50293_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 678960 INFO  (zkCallback-1647-thread-1-processing-n:127.0.0.1:50294_solr) [n:127.0.0.1:50294_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 678960 INFO  (jetty-launcher-1617-thread-3) [n:127.0.0.1:50289_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 678961 INFO  (zkCallback-1649-thread-1-processing-n:127.0.0.1:50293_solr) [n:127.0.0.1:50293_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 678961 INFO  (zkCallback-1639-thread-1-processing-n:127.0.0.1:50289_solr) [n:127.0.0.1:50289_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 678962 INFO  (zkCallback-1647-thread-1-processing-n:127.0.0.1:50294_solr) [n:127.0.0.1:50294_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 678963 INFO  (zkCallback-1646-thread-1-processing-n:127.0.0.1:50296_solr) [n:127.0.0.1:50296_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 679004 INFO  (zkCallback-1645-thread-1-processing-n:127.0.0.1:50290_solr) [n:127.0.0.1:50290_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 679004 INFO  (zkCallback-1648-thread-1-processing-n:127.0.0.1:50295_solr) [n:127.0.0.1:50295_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 679005 INFO  (jetty-launcher-1617-thread-4) [n:127.0.0.1:50293_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 679008 INFO  (jetty-launcher-1617-thread-3) [n:127.0.0.1:50289_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50289_solr
   [junit4]   2> 679010 INFO  (jetty-launcher-1617-thread-4) [n:127.0.0.1:50293_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50293_solr
   [junit4]   2> 679011 INFO  (zkCallback-1639-thread-1-processing-n:127.0.0.1:50289_solr) [n:127.0.0.1:50289_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 679012 INFO  (zkCallback-1648-thread-1-processing-n:127.0.0.1:50295_solr) [n:127.0.0.1:50295_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 679012 INFO  (zkCallback-1649-thread-1-processing-n:127.0.0.1:50293_solr) [n:127.0.0.1:50293_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 679012 INFO  (zkCallback-1646-thread-1-processing-n:127.0.0.1:50296_solr) [n:127.0.0.1:50296_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 679012 INFO  (zkCallback-1647-thread-1-processing-n:127.0.0.1:50294_solr) [n:127.0.0.1:50294_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 679012 INFO  (zkCallback-1645-thread-1-processing-n:127.0.0.1:50290_solr) [n:127.0.0.1:50290_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 679040 INFO  (zkCallback-1645-thread-1-processing-n:127.0.0.1:50290_solr) [n:127.0.0.1:50290_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 679040 INFO  (zkCallback-1649-thread-1-processing-n:127.0.0.1:50293_solr) [n:127.0.0.1:50293_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 679040 INFO  (zkCallback-1646-thread-1-processing-n:127.0.0.1:50296_solr) [n:127.0.0.1:50296_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 679040 INFO  (zkCallback-1639-thread-1-processing-n:127.0.0.1:50289_solr) [n:127.0.0.1:50289_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 679040 INFO  (zkCallback-1648-thread-1-processing-n:127.0.0.1:50295_solr) [n:127.0.0.1:50295_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 679041 INFO  (zkCallback-1647-thread-1-processing-n:127.0.0.1:50294_solr) [n:127.0.0.1:50294_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 679045 INFO  (jetty-launcher-1617-thread-6) [n:127.0.0.1:50290_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50290_solr
   [junit4]   2> 679048 INFO  (zkCallback-1648-thread-1-processing-n:127.0.0.1:50295_solr) [n:127.0.0.1:50295_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (6)
   [junit4]   2> 679048 INFO  (zkCallback-1646-thread-1-processing-n:127.0.0.1:50296_solr) [n:127.0.0.1:50296_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (6)
   [junit4]   2> 679048 INFO  (zkCallback-1647-thread-1-processing-n:127.0.0.1:50294_solr) [n:127.0.0.1:50294_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (6)
   [junit4]   2> 679048 INFO  (zkCallback-1639-thread-1-processing-n:127.0.0.1:50289_solr) [n:127.0.0.1:50289_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (6)
   [junit4]   2> 679049 INFO  (zkCallback-1649-thread-1-processing-n:127.0.0.1:50293_solr) [n:127.0.0.1:50293_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (6)
   [junit4]   2> 679049 INFO  (zkCallback-1645-thread-1-processing-n:127.0.0.1:50290_solr) [n:127.0.0.1:50290_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (6)
   [junit4]   2> 679125 INFO  (jetty-launcher-1617-thread-3) [n:127.0.0.1:50289_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50289.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3befc08c
   [junit4]   2> 679128 INFO  (jetty-launcher-1617-thread-3) [n:127.0.0.1:50289_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50289.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3befc08c
   [junit4]   2> 679128 INFO  (jetty-launcher-1617-thread-3) [n:127.0.0.1:50289_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50289.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3befc08c
   [junit4]   2> 679128 INFO  (jetty-launcher-1617-thread-3) [n:127.0.0.1:50289_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_20D3DA769E1BC5EF-001/tempDir-001/node2/.
   [junit4]   2> 679130 INFO  (jetty-launcher-1617-thread-6) [n:127.0.0.1:50290_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50290.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3befc08c
   [junit4]   2> 679132 INFO  (jetty-launcher-1617-thread-6) [n:127.0.0.1:50290_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50290.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3befc08c
   [junit4]   2> 679132 INFO  (jetty-launcher-1617-thread-6) [n:127.0.0.1:50290_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50290.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3befc08c
   [junit4]   2> 679133 INFO  (jetty-launcher-1617-thread-6) [n:127.0.0.1:50290_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_20D3DA769E1BC5EF-001/tempDir-001/node6/.
   [junit4]   2> 679137 INFO  (jetty-launcher-1617-thread-2) [n:127.0.0.1:50296_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50296.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3befc08c
   [junit4]   2> 679140 INFO  (jetty-launcher-1617-thread-2) [n:127.0.0.1:50296_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50296.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3befc08c
   [junit4]   2> 679140 INFO  (jetty-launcher-1617-thread-2) [n:127.0.0.1:50296_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50296.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3befc08c
   [junit4]   2> 679141 INFO  (jetty-launcher-1617-thread-2) [n:127.0.0.1:50296_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_20D3DA769E1BC5EF-001/tempDir-001/node1/.
   [junit4]   2> 679146 INFO  (jetty-launcher-1617-thread-4) [n:127.0.0.1:50293_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50293.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3befc08c
   [junit4]   2> 679148 INFO  (jetty-launcher-1617-thread-4) [n:127.0.0.1:50293_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50293.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3befc08c
   [junit4]   2> 679148 INFO  (jetty-launcher-1617-thread-4) [n:127.0.0.1:50293_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50293.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3befc08c
   [junit4]   2> 679149 INFO  (jetty-launcher-1617-thread-4) [n:127.0.0.1:50293_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_20D3DA769E1BC5EF-001/tempDir-001/node4/.
   [junit4]   2> 679162 INFO  (jetty-launcher-1617-thread-1) [n:127.0.0.1:50295_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50295.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3befc08c
   [junit4]   2> 679177 INFO  (jetty-launcher-1617-thread-1) [n:127.0.0.1:50295_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50295.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3befc08c
   [junit4]   2> 679177 INFO  (jetty-launcher-1617-thread-1) [n:127.0.0.1:50295_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50295.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3befc08c
   [junit4]   2> 679178 INFO  (jetty-launcher-1617-thread-1) [n:127.0.0.1:50295_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_20D3DA769E1BC5EF-001/tempDir-001/node3/.
   [junit4]   2> 679208 INFO  (jetty-launcher-1617-thread-5) [n:127.0.0.1:50294_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50294.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3befc08c
   [junit4]   2> 679211 INFO  (jetty-launcher-1617-thread-5) [n:127.0.0.1:50294_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50294.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3befc08c
   [junit4]   2> 679211 INFO  (jetty-launcher-1617-thread-5) [n:127.0.0.1:50294_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50294.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3befc08c
   [junit4]   2> 679211 INFO  (jetty-launcher-1617-thread-5) [n:127.0.0.1:50294_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_20D3DA769E1BC5EF-001/tempDir-001/node5/.
   [junit4]   2> 679333 INFO  (SUITE-ReplaceNodeTest-seed#[20D3DA769E1BC5EF]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (6)
   [junit4]   2> 679334 INFO  (SUITE-ReplaceNodeTest-seed#[20D3DA769E1BC5EF]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:50287/solr ready
   [junit4]   2> 679374 INFO  (TEST-ReplaceNodeTest.test-seed#[20D3DA769E1BC5EF]) [    ] o.a.s.SolrTestCaseJ4 ###Starting test
   [junit4]   2> 679377 INFO  (TEST-ReplaceNodeTest.test-seed#[20D3DA769E1BC5EF]) [    ] o.a.s.c.ReplaceNodeTest total_jettys: 6
   [junit4]   2> 679377 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15d8679c8680015, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 679380 INFO  (qtp1792046842-9098) [n:127.0.0.1:50295_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params pullReplicas=1&replicationFactor=1&collection.configName=conf1&maxShardsPerNode=3&name=replacenodetest_coll&nrtReplicas=1&action=CREATE&numShards=5&tlogReplicas=0&createNodeSet=127.0.0.1:50294_solr,127.0.0.1:50290_solr,127.0.0.1:50296_solr,127.0.0.1:50289_solr,127.0.0.1:50295_solr&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 679382 INFO  (OverseerThreadFactory-4276-thread-1-processing-n:127.0.0.1:50290_solr) [n:127.0.0.1:50290_solr    ] o.a.s.c.CreateCollectionCmd Create collection replacenodetest_coll
   [junit4]   2> 679502 INFO  (OverseerStateUpdate-98382624483442702-127.0.0.1:50290_solr-n_0000000000) [n:127.0.0.1:50290_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"replacenodetest_coll",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"replacenodetest_coll_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:50289/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 679507 INFO  (OverseerStateUpdate-98382624483442702-127.0.0.1:50290_solr-n_0000000000) [n:127.0.0.1:50290_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"replacenodetest_coll",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"replacenodetest_coll_shard1_replica_p2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:50290/solr",
   [junit4]   2>   "type":"PULL"} 
   [junit4]   2> 679511 INFO  (OverseerStateUpdate-98382624483442702-127.0.0.1:50290_solr-n_0000000000) [n:127.0.0.1:50290_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"replacenodetest_coll",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"replacenodetest_coll_shard2_replica_n4",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:50294/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 679516 INFO  (OverseerStateUpdate-98382624483442702-127.0.0.1:50290_solr-n_0000000000) [n:127.0.0.1:50290_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"replacenodetest_coll",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"replacenodetest_coll_shard2_replica_p6",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:50295/solr",
   [junit4]   2>   "type":"PULL"} 
   [junit4]   2> 679521 INFO  (OverseerStateUpdate-98382624483442702-127.0.0.1:50290_solr-n_0000000000) [n:127.0.0.1:50290_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"replacenodetest_coll",
   [junit4]   2>   "shard":"shard3",
   [junit4]   2>   "core":"replacenodetest_coll_shard3_replica_n8",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:50296/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 679525 INFO  (OverseerStateUpdate-98382624483442702-127.0.0.1:50290_solr-n_0000000000) [n:127.0.0.1:50290_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"replacenodetest_coll",
   [junit4]   2>   "shard":"shard3",
   [junit4]   2>   "core":"replacenodetest_coll_shard3_replica_p10",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:50289/solr",
   [junit4]   2>   "type":"PULL"} 
   [junit4]   2> 679530 INFO  (OverseerStateUpdate-98382624483442702-127.0.0.1:50290_solr-n_0000000000) [n:127.0.0.1:50290_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"replacenodetest_coll",
   [junit4]   2>   "shard":"shard4",
   [junit4]   2>   "core":"replacenodetest_coll_shard4_replica_n12",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:50290/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 679534 INFO  (OverseerStateUpdate-98382624483442702-127.0.0.1:50290_solr-n_0000000000) [n:127.0.0.1:50290_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"replacenodetest_coll",
   [junit4]   2>   "shard":"shard4",
   [junit4]   2>   "core":"replacenodetest_coll_shard4_replica_p14",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:50294/solr",
   [junit4]   2>   "type":"PULL"} 
   [junit4]   2> 679540 INFO  (OverseerStateUpdate-98382624483442702-127.0.0.1:50290_solr-n_0000000000) [n:127.0.0.1:50290_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"replacenodetest_coll",
   [junit4]   2>   "shard":"shard5",
   [junit4]   2>   "core":"replacenodetest_coll_shard5_replica_n16",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:50295/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 679544 INFO  (OverseerStateUpdate-98382624483442702-127.0.0.1:50290_solr-n_0000000000) [n:127.0.0.1:50290_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"replacenodetest_coll",
   [junit4]   2>   "shard":"shard5",
   [junit4]   2>   "core":"replacenodetest_coll_shard5_replica_p18",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:50296/solr",
   [junit4]   2>   "type":"PULL"} 
   [junit4]   2> 679746 INFO  (qtp1792046842-9099) [n:127.0.0.1:50295_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node9&collection.configName=conf1&newCollection=true&name=replacenodetest_coll_shard2_replica_p6&action=CREATE&numShards=5&collection=replacenodetest_coll&shard=shard2&wt=javabin&version=2&replicaType=PULL
   [junit4]   2> 679747 INFO  (qtp1792046842-9099) [n:127.0.0.1:50295_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 679748 INFO  (qtp1257807132-9073) [n:127.0.0.1:50294_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node17&collection.configName=conf1&newCollection=true&name=replacenodetest_coll_shard4_replica_p14&action=CREATE&numShards=5&collection=replacenodetest_coll&shard=shard4&wt=javabin&version=2&replicaType=PULL
   [junit4]   2> 679751 INFO  (qtp1257807132-9073) [n:127.0.0.1:50294_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 679751 INFO  (qtp1792046842-9100) [n:127.0.0.1:50295_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node19&collection.configName=conf1&newCollection=true&name=replacenodetest_coll_shard5_replica_n16&action=CREATE&numShards=5&collection=replacenodetest_coll&shard=shard5&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 679752 INFO  (qtp1582697236-9054) [n:127.0.0.1:50290_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node15&collection.configName=conf1&newCollection=true&name=replacenodetest_coll_shard4_replica_n12&action=CREATE&numShards=5&collection=replacenodetest_coll&shard=shard4&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 679752 INFO  (qtp2018161192-9050) [n:127.0.0.1:50296_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node11&collection.configName=conf1&newCollection=true&name=replacenodetest_coll_shard3_replica_n8&action=CREATE&numShards=5&collection=replacenodetest_coll&shard=shard3&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 679752 INFO  (qtp2018161192-9051) [n:127.0.0.1:50296_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node20&collection.configName=conf1&newCollection=true&name=replacenodetest_coll_shard5_replica_p18&action=CREATE&numShards=5&collection=replacenodetest_coll&shard=shard5&wt=javabin&version=2&replicaType=PULL
   [junit4]   2> 679752 INFO  (qtp2018161192-9050) [n:127.0.0.1:50296_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 679752 INFO  (qtp1582697236-9054) [n:127.0.0.1:50290_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 679752 INFO  (qtp1582697236-9056) [n:127.0.0.1:50290_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node5&collection.configName=conf1&newCollection=true&name=replacenodetest_coll_shard1_replica_p2&action=CREATE&numShards=5&collection=replacenodetest_coll&shard=shard1&wt=javabin&version=2&replicaType=PULL
   [junit4]   2> 679753 INFO  (qtp1464304042-9063) [n:127.0.0.1:50289_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf1&newCollection=true&name=replacenodetest_coll_shard1_replica_n1&action=CREATE&numShards=5&collection=replacenodetest_coll&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 679753 INFO  (qtp1464304042-9063) [n:127.0.0.1:50289_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 679754 INFO  (qtp1464304042-9064) [n:127.0.0.1:50289_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node13&collection.configName=conf1&newCollection=true&name=replacenodetest_coll_shard3_replica_p10&action=CREATE&numShards=5&collection=replacenodetest_coll&shard=shard3&wt=javabin&version=2&replicaType=PULL
   [junit4]   2> 679755 INFO  (qtp1257807132-9076) [n:127.0.0.1:50294_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node7&collection.configName=conf1&newCollection=true&name=replacenodetest_coll_shard2_replica_n4&action=CREATE&numShards=5&collection=replacenodetest_coll&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 679882 INFO  (zkCallback-1639-thread-1-processing-n:127.0.0.1:50289_solr) [n:127.0.0.1:50289_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/replacenodetest_coll/state.json] for collection [replacenodetest_coll] has occurred - updating... (live nodes size: [6])
   [junit4]   2> 679882 INFO  (zkCallback-1647-thread-1-processing-n:127.0.0.1:50294_solr) [n:127.0.0.1:50294_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/replacenodetest_coll/state.json] for collection [replacenodetest_coll] has occurred - updating... (live nodes size: [6])
   [junit4]   2> 679882 INFO  (zkCallback-1646-thread-1-processing-n:127.0.0.1:50296_solr) [n:127.0.0.1:50296_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/replacenodetest_coll/state.json] for collection [replacenodetest_coll] has occurred - updating... (live nodes size: [6])
   [junit4]   2> 679882 INFO  (zkCallback-1645-thread-1-processing-n:127.0.0.1:50290_solr) [n:127.0.0.1:50290_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/replacenodetest_coll/state.json] for collection [replacenodetest_coll] has occurred - updating... (live nodes size: [6])
   [junit4]   2> 679882 INFO  (zkCallback-1648-thread-1-processing-n:127.0.0.1:50295_solr) [n:127.0.0.1:50295_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/replacenodetest_coll/state.json] for collection [replacenodetest_coll] has occurred - updating... (live nodes size: [6])
   [junit4]   2> 679883 INFO  (zkCallback-1639-thread-2-processing-n:127.0.0.1:50289_solr) [n:127.0.0.1:50289_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/replacenodetest_coll/state.json] for collection [replacenodetest_coll] has occurred - updating... (live nodes size: [6])
   [junit4]   2> 679883 INFO  (zkCallback-1645-thread-2-processing-n:127.0.0.1:50290_solr) [n:127.0.0.1:50290_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/replacenodetest_coll/state.json] for collection [replacenodetest_coll] has occurred - updating... (live nodes size: [6])
   [junit4]   2> 679885 INFO  (zkCallback-1648-thread-2-processing-n:127.0.0.1:50295_solr) [n:127.0.0.1:50295_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/replacenodetest_coll/state.json] for collection [replacenodetest_coll] has occurred - updating... (live nodes size: [6])
   [junit4]   2> 679886 INFO  (zkCallback-1647-thread-1-processing-n:127.0.0.1:50294_solr) [n:127.0.0.1:50294_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/replacenodetest_coll/state.json] for collection [replacenodetest_coll] has occurred - updating... (live nodes size: [6])
   [junit4]   2> 679886 INFO  (zkCallback-1647-thread-2-processing-n:127.0.0.1:50294_solr) [n:127.0.0.1:50294_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/replacenodetest_coll/state.json] for collection [replacenodetest_coll] has occurred - updating... (live nodes size: [6])
   [junit4]   2> 680778 INFO  (qtp1582697236-9054) [n:127.0.0.1:50290_solr c:replacenodetest_coll s:shard4 r:core_node15 x:replacenodetest_coll_shard4_replica_n12] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 680778 INFO  (qtp2018161192-9050) [n:127.0.0.1:50296_solr c:replacenodetest_coll s:shard3 r:core_node11 x:replacenodetest_coll_shard3_replica_n8] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 680778 INFO  (qtp1792046842-9099) [n:127.0.0.1:50295_solr c:replacenodetest_coll s:shard2 r:core_node9 x:replacenodetest_coll_shard2_replica_p6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 680798 INFO  (qtp1257807132-9073) [n:127.0.0.1:50294_solr c:replacenodetest_coll s:shard4 r:core_node17 x:replacenodetest_coll_shard4_replica_p14] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 680799 INFO  (qtp2018161192-9050) [n:127.0.0.1:50296_solr c:replacenodetest_coll s:shard3 r:core_node11 x:replacenodetest_coll_shard3_replica_n8] o.a.s.s.IndexSchema [replacenodetest_coll_shard3_replica_n8] Schema name=cloud-dynamic
   [junit4]   2> 680800 INFO  (qtp1582697236-9056) [n:127.0.0.1:50290_solr c:replacenodetest_coll s:shard1 r:core_node5 x:replacenodetest_coll_shard1_replica_p2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 680802 INFO  (qtp1792046842-9099) [n:127.0.0.1:50295_solr c:replacenodetest_coll s:shard2 r:core_node9 x:replacenodetest_coll_shard2_replica_p6] o.a.s.s.IndexSchema [replacenodetest_coll_shard2_replica_p6] Schema name=cloud-dynamic
   [junit4]   2> 680799 INFO  (qtp1582697236-9054) [n:127.0.0.1:50290_solr c:replacenodetest_coll s:shard4 r:core_node15 x:replacenodetest_coll_shard4_replica_n12] o.a.s.s.IndexSchema [replacenodetest_coll_shard4_replica_n12] Schema name=cloud-dynamic
   [junit4]   2> 680831 INFO  (qtp2018161192-9051) [n:127.0.0.1:50296_solr c:replacenodetest_coll s:shard5 r:core_node20 x:replacenodetest_coll_shard5_replica_p18] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 680832 INFO  (qtp1792046842-9100) [n:127.0.0.1:50295_solr c:replacenodetest_coll s:shard5 r:core_node19 x:replacenodetest_coll_shard5_replica_n16] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 680836 INFO  (qtp1464304042-9064) [n:127.0.0.1:50289_solr c:replacenodetest_coll s:shard3 r:core_node13 x:replacenodetest_coll_shard3_replica_p10] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 680836 INFO  (qtp1582697236-9054) [n:127.0.0.1:50290_solr c:replacenodetest_coll s:shard4 r:core_node15 x:replacenodetest_coll_shard4_replica_n12] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
   [junit4]   2> 680837 INFO  (qtp1582697236-9054) [n:127.0.0.1:50290_solr c:replacenodetest_coll s:shard4 r:core_node15 x:replacenodetest_coll_shard4_replica_n12] o.a.s.c.CoreContainer Creating SolrCore 'replacenodetest_coll_shard4_replica_n12' using configuration from collection replacenodetest_coll, trusted=true
   [junit4]   2> 680837 INFO  (qtp1582697236-9054) [n:127.0.0.1:50290_solr c:replacenodetest_coll s:shard4 r:core_node15 x:replacenodetest_coll_shard4_replica_n12] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50290.solr.core.replacenodetest_coll.shard4.replica_n12' (registry 'solr.core.replacenodetest_coll.shard4.replica_n12') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3befc08c
   [junit4]   2> 680847 INFO  (qtp1582697236-9054) [n:127.0.0.1:50290_solr c:replacenodetest_coll s:shard4 r:core_node15 x:replacenodetest_coll_shard4_replica_n12] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 680847 INFO  (qtp1582697236-9054) [n:127.0.0.1:50290_solr c:replacenodetest_coll s:shard4 r:core_node15 x:replacenodetest_coll_shard4_replica_n12] o.a.s.c.SolrCore [[replacenodetest_coll_shard4_replica_n12] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_20D3DA769E1BC5EF-001/tempDir-001/node6/replacenodetest_coll_shard4_replica_n12], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_20D3DA769E1BC5EF-001/tempDir-001/node6/./replacenodetest_coll_shard4_replica_n12/data/]
   [junit4]   2> 680851 INFO  (qtp1464304042-9063) [n:127.0.0.1:50289_solr c:replacenodetest_coll s:shard1 r:core_node3 x:replacenodetest_coll_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 680851 INFO  (qtp2018161192-9050) [n:127.0.0.1:50296_solr c:replacenodetest_coll s:shard3 r:core_node11 x:replacenodetest_coll_shard3_replica_n8] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
   [junit4]   2> 680851 INFO  (qtp2018161192-9050) [n:127.0.0.1:50296_solr c:replacenodetest_coll s:shard3 r:core_node11 x:replacenodetest_coll_shard3_replica_n8] o.a.s.c.CoreContainer Creating SolrCore 'replacenodetest_coll_shard3_replica_n8' using configuration from collection replacenodetest_coll, trusted=true
   [junit4]   2> 680851 INFO  (qtp2018161192-9050) [n:127.0.0.1:50296_solr c:replacenodetest_coll s:shard3 r:core_node11 x:replacenodetest_coll_shard3_replica_n8] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50296.solr.core.replacenodetest_coll.shard3.replica_n8' (registry 'solr.core.replacenodetest_coll.shard3.replica_n8') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3befc08c
   [junit4]   2> 680851 INFO  (qtp2018161192-9050) [n:127.0.0.1:50296_solr c:replacenodetest_coll s:shard3 r:core_node11 x:replacenodetest_coll_shard3_replica_n8] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 680852 INFO  (qtp2018161192-9050) [n:127.0.0.1:50296_solr c:replacenodetest_coll s:shard3 r:core_node11 x:replacenodetest_coll_shard3_replica_n8] o.a.s.c.SolrCore [[replacenodetest_coll_shard3_replica_n8] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_20D3DA769E1BC5EF-001/tempDir-001/node1/replacenodetest_coll_shard3_replica_n8], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_20D3DA769E1BC5EF-001/tempDir-001/node1/./replacenodetest_coll_shard3_replica_n8/data/]
   [junit4]   2> 680853 INFO  (qtp1582697236-9056) [n:127.0.0.1:50290_solr c:replacenodetest_coll s:shard1 r:core_node5 x:replacenodetest_coll_shard1_replica_p2] o.a.s.s.IndexSchema [replacenodetest_coll_shard1_replica_p2] Schema name=cloud-dynamic
   [junit4]   2> 680854 INFO  (qtp1792046842-9099) [n:127.0.0.1:50295_solr c:replacenodetest_coll s:shard2 r:core_node9 x:replacenodetest_coll_shard2_replica_p6] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
   [junit4]   2> 680854 INFO  (qtp1792046842-9099) [n:127.0.0.1:50295_solr c:replacenodetest_coll s:shard2 r:core_node9 x:replacenodetest_coll_shard2_replica_p6] o.a.s.c.CoreContainer Creating SolrCore 'replacenodetest_coll_shard2_replica_p6' using configuration from collection replacenodetest_coll, trusted=true
   [junit4]   2> 680854 INFO  (qtp1792046842-9099) [n:127.0.0.1:50295_solr c:replacenodetest_coll s:shard2 r:core_node9 x:replacenodetest_coll_shard2_replica_p6] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50295.solr.core.replacenodetest_coll.shard2.replica_p6' (registry 'solr.core.replacenodetest_coll.shard2.replica_p6') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3befc08c
   [junit4]   2> 680854 INFO  (qtp1792046842-9099) [n:127.0.0.1:50295_solr c:replacenodetest_coll s:shard2 r:core_node9 x:replacenodetest_coll_shard2_replica_p6] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 680854 INFO  (qtp1792046842-9099) [n:127.0.0.1:50295_solr c:replacenodetest_coll s:shard2 r:core_node9 x:replacenodetest_coll_shard2_replica_p6] o.a.s.c.SolrCore [[replacenodetest_coll_shard2_replica_p6] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_20D3DA769E1BC5EF-001/tempDir-001/node3/replacenodetest_coll_shard2_replica_p6], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_20D3DA769E1BC5EF-001/tempDir-001/node3/./replacenodetest_coll_shard2_replica_p6/data/]
   [junit4]   2> 680857 INFO  (qtp1257807132-9076) [n:127.0.0.1:50294_solr c:replacenodetest_coll s:shard2 r:core_node7 x:replacenodetest_coll_shard2_replica_n4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 680872 INFO  (qtp1257807132-9073) [n:127.0.0.1:50294_solr c:replacenodetest_coll s:shard4 r:core_node17 x:replacenodetest_coll_shard4_replica_p14] o.a.s.s.IndexSchema [replacenodetest_coll_shard4_replica_p14] Schema name=cloud-dynamic
   [junit4]   2> 680878 INFO  (qtp1792046842-9100) [n:127.0.0.1:50295_solr c:replacenodetest_coll s:shard5 r:core_node19 x:replacenodetest_coll_shard5_replica_n16] o.a.s.s.IndexSchema [replacenodetest_coll_shard5_replica_n16] Schema name=cloud-dynamic
   [junit4]   2> 680878 INFO  (qtp1582697236-9056) [n:127.0.0.1:50290_solr c:replacenodetest_coll s:shard1 r:core_node5 x:replacenodetest_coll_shard1_replica_p2] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
   [junit4]   2> 680878 INFO  (qtp1582697236-9056) [n:127.0.0.1:50290_solr c:replacenodetest_coll s:shard1 r:core_node5 x:replacenodetest_coll_shard1_replica_p2] o.a.s.c.CoreContainer Creating SolrCore 'replacenodetest_coll_shard1_replica_p2' using configuration from collection replacenodetest_coll, trusted=true
   [junit4]   2> 680878 INFO  (qtp1582697236-9056) [n:127.0.0.1:50290_solr c:replacenodetest_coll s:shard1 r:core_node5 x:replacenodetest_coll_shard1_replica_p2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50290.solr.core.replacenodetest_coll.shard1.replica_p2' (registry 'solr.core.replacenodetest_coll.shard1.replica_p2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3befc08c
   [junit4]   2> 680878 INFO  (qtp1582697236-9056) [n:127.0.0.1:50290_solr c:replacenodetest_coll s:shard1 r:core_node5 x:replacenodetest_coll_shard1_replica_p2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 680878 INFO  (qtp1582697236-9056) [n:127.0.0.1:50290_solr c:replacenodetest_coll s:shard1 r:core_node5 x:replacenodetest_coll_shard1_replica_p2] o.a.s.c.SolrCore [[replacenodetest_coll_shard1_replica_p2] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_20D3DA769E1BC5EF-001/tempDir-001/node6/replacenodetest_coll_shard1_replica_p2], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_20D3DA769E1BC5EF-001/tempDir-001/node6/./replacenodetest_coll_shard1_replica_p2/data/]
   [junit4]   2> 680894 INFO  (qtp2018161192-9051) [n:127.0.0.1:50296_solr c:replacenodetest_coll s:shard5 r:core_node20 x:replacenodetest_coll_shard5_replica_p18] o.a.s.s.IndexSchema [replacenodetest_coll_shard5_replica_p18] Schema name=cloud-dynamic
   [junit4]   2> 680894 INFO  (qtp1464304042-9064) [n:127.0.0.1:50289_solr c:replacenodetest_coll s:shard3 r:core_node13 x:replacenodetest_coll_shard3_replica_p10] o.a.s.s.IndexSchema [replacenodetest_coll_shard3_replica_p10] Schema name=cloud-dynamic
   [junit4]   2> 680895 INFO  (qtp1257807132-9073) [n:127.0.0.1:50294_solr c:replacenodetest_coll s:shard4 r:core_node17 x:replacenodetest_coll_shard4_replica_p14] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
   [junit4]   2> 680895 INFO  (qtp1257807132-9073) [n:127.0.0.1:50294_solr c:replacenodetest_coll s:shard4 r:core_node17 x:replacenodetest_coll_shard4_replica_p14] o.a.s.c.CoreContainer Creating SolrCore 'replacenodetest_coll_shard4_replica_p14' using configuration from collection replacenodetest_coll, trusted=true
   [junit4]   2> 680896 INFO  (qtp1257807132-9073) [n:127.0.0.1:50294_solr c:replacenodetest_coll s:shard4 r:core_node17 x:replacenodetest_coll_shard4_replica_p14] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50294.solr.core.replacenodetest_coll.shard4.replica_p14' (registry 'solr.core.replacenodetest_coll.shard4.replica_p14') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3befc08c
   [junit4]   2> 680896 INFO  (qtp1257807132-9073) [n:127.0.0.1:50294_solr c:replacenodetest_coll s:shard4 r:core_node17 x:replacenodetest_coll_shard4_replica_p14] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 680896 INFO  (qtp1257807132-9073) [n:127.0.0.1:50294_solr c:replacenodetest_coll s:shard4 r:core_node17 x:replacenodetest_coll_shard4_replica_p14] o.a.s.c.SolrCore [[replacenodetest_coll_shard4_replica_p14] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_20D3DA769E1BC5EF-001/tempDir-001/node5/replacenodetest_coll_shard4_replica_p14], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_20D3DA769E1BC5EF-001/tempDir-001/node5/./replacenodetest_coll_shard4_replica_p14/data/]
   [junit4]   2> 680918 INFO  (qtp1464304042-9064) [n:127.0.0.1:50289_solr c:replacenodetest_coll s:shard3 r:core_node13 x:replacenodetest_coll_shard3_replica_p10] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
   [junit4]   2> 680918 INFO  (qtp1464304042-9064) [n:127.0.0.1:50289_solr c:replacenodetest_coll s:shard3 r:core_node13 x:replacenodetest_coll_shard3_replica_p10] o.a.s.c.CoreContainer Creating SolrCore 'replacenodetest_coll_shard3_replica_p10' using configuration from collection replacenodetest_coll, trusted=true
   [junit4]   2> 680919 INFO  (qtp2018161192-9051) [n:127.0.0.1:50296_solr c:replacenodetest_coll s:shard5 r:core_node20 x:replacenodetest_coll_shard5_replica_p18] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
   [junit4]   2> 680919 INFO  (qtp2018161192-9051) [n:127.0.0.1:50296_solr c:replacenodetest_coll s:shard5 r:core_node20 x:replacenodetest_coll_shard5_replica_p18] o.a.s.c.CoreContainer Creating SolrCore 'replacenodetest_coll_shard5_replica_p18' using configuration from collection replacenodetest_coll, trusted=true
   [junit4]   2> 680919 INFO  (qtp1464304042-9064) [n:127.0.0.1:50289_solr c:replacenodetest_coll s:shard3 r:core_node13 x:replacenodetest_coll_shard3_replica_p10] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50289.solr.core.replacenodetest_coll.shard3.replica_p10' (registry 'solr.core.replacenodetest_coll.shard3.replica_p10') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3befc08c
   [junit4]   2> 680919 INFO  (qtp1464304042-9064) [n:127.0.0.1:50289_solr c:replacenodetest_coll s:shard3 r:core_node13 x:replacenodetest_coll_shard3_replica_p10] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 680919 INFO  (qtp1464304042-9064) [n:127.0.0.1:50289_solr c:replacenodetest_coll s:shard3 r:core_node13 x:replacenodetest_coll_shard3_replica_p10] o.a.s.c.SolrCore [[replacenodetest_coll_shard3_replica_p10] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_20D3DA769E1BC5EF-001/tempDir-001/node2/replacenodetest_coll_shard3_replica_p10], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_20D3DA769E1BC5EF-001/tempDir-001/node2/./replacenodetest_coll_shard3_replica_p10/data/]
   [junit4]   2> 680919 INFO  (qtp2018161192-9051) [n:127.0.0.1:50296_solr c:replacenodetest_coll s:shard5 r:core_node20 x:replacenodetest_coll_shard5_replica_p18] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50296.solr.core.replacenodetest_coll.shard5.replica_p18' (registry 'solr.core.replacenodetest_coll.shard5.replica_p18') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3befc08c
   [junit4]   2> 680919 INFO  (qtp2018161192-9051) [n:127.0.0.1:50296_solr c:replacenodetest_coll s:shard5 r:core_node20 x:replacenodetest_coll_shard5_replica_p18] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 680919 INFO  (qtp2018161192-9051) [n:127.0.0.1:50296_solr c:replacenodetest_coll s:shard5 r:core_node20 x:replacenodetest_coll_shard5_replica_p18] o.a.s.c.SolrCore [[replacenodetest_coll_shard5_replica_p18] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_20D3DA769E1BC5EF-001/tempDir-001/node1/replacenodetest_coll_shard5_replica_p18], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_20D3DA769E1BC5EF-001/tempDir-001/node1/./replacenodetest_coll_shard5_replica_p18/data/]
   [junit4]   2> 680922 INFO  (qtp1792046842-9100) [n:127.0.0.1:50295_solr c:replacenodetest_coll s:shard5 r:core_node19 x:replacenodetest_coll_shard5_replica_n16] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
   [junit4]   2> 680922 INFO  (qtp1792046842-9100) [n:127.0.0.1:50295_solr c:replacenodetest_coll s:shard5 r:core_node19 x:replacenodetest_coll_shard5_replica_n16] o.a.s.c.CoreContainer Creating SolrCore 'replacenodetest_coll_shard5_replica_n16' using configuration from collection replacenodetest_coll, trusted=true
   [junit4]   2> 680922 INFO  (qtp1792046842-9100) [n:127.0.0.1:50295_solr c:replacenodetest_coll s:shard5 r:core_node19 x:replacenodetest_coll_shard5_replica_n16] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50295.solr.core.replacenodetest_coll.shard5.replica_n16' (registry 'solr.core.replacenodetest_coll.shard5.replica_n16') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3befc08c
   [junit4]   2> 680922 INFO  (qtp1792046842-9100) [n:127.0.0.1:50295_solr c:replacenodetest_coll s:shard5 r:core_node19 x:replacenodetest_coll_shard5_replica_n16] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 680922 INFO  (qtp1792046842-9100) [n:127.0.0.1:50295_solr c:replacenodetest_coll s:shard5 r:core_node19 x:replacenodetest_coll_shard5_replica_n16] o.a.s.c.SolrCore [[replacenodetest_coll_shard5_replica_n16] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_20D3DA769E1BC5EF-001/tempDir-001/node3/replacenodetest_coll_shard5_replica_n16], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_20D3DA769E1BC5EF-001/tempDir-001/node3/./replacenodetest_coll_shard5_replica_n16/data/]
   [junit4]   2> 680922 INFO  (qtp1464304042-9063) [n:127.0.0.1:50289_solr c:replacenodetest_coll s:shard1 r:core_node3 x:replacenodetest_coll_shard1_replica_n1] o.a.s.s.IndexSchema [replacenodetest_coll_shard1_replica_n1] Schema name=cloud-dynamic
   [junit4]   2> 680922 INFO  (qtp1257807132-9076) [n:127.0.0.1:50294_solr c:replacenodetest_coll s:shard2 r:core_node7 x:replacenodetest_coll_shard2_replica_n4] o.a.s.s.IndexSchema [replacenodetest_coll_shard2_replica_n4] Schema name=cloud-dynamic
   [junit4]   2> 680943 INFO  (qtp1257807132-9076) [n:127.0.0.1:50294_solr c:replacenodetest_coll s:shard2 r:core_node7 x:replacenodetest_coll_shard2_replica_n4] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
   [junit4]   2> 680944 INFO  (qtp1257807132-9076) [n:127.0.0.1:50294_solr c:replacenodetest_coll s:shard2 r:core_node7 x:replacenodetest_coll_shard2_replica_n4] o.a.s.c.CoreContainer Creating SolrCore 'replacenodetest_coll_shard2_replica_n4' using configuration from collection replacenodetest_coll, trusted=true
   [junit4]   2> 680944 INFO  (qtp1257807132-9076) [n:127.0.0.1:50294_solr c:replacenodetest_coll s:shard2 r:core_node7 x:replacenodetest_coll_shard2_replica_n4] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50294.solr.core.replacenodetest_coll.shard2.replica_n4' (registry 'solr.core.replacenodetest_coll.shard2.replica_n4') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3befc08c
   [junit4]   2> 680944 INFO  (qtp1257807132-9076) [n:127.0.0.1:50294_solr c:replacenodetest_coll s:shard2 r:core_node7 x:replacenodetest_coll_shard2_replica_n4] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 680944 INFO  (qtp1257807132-9076) [n:127.0.0.1:50294_solr c:replacenodetest_coll s:shard2 r:core_node7 x:replacenodetest_coll_shard2_replica_n4] o.a.s.c.SolrCore [[replacenodetest_coll_shard2_replica_n4] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_20D3DA769E1BC5EF-001/tempDir-001/node5/replacenodetest_coll_shard2_replica_n4], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_20D3DA769E1BC5EF-001/tempDir-001/node5/./replacenodetest_coll_shard2_replica_n4/data/]
   [junit4]   2> 680949 INFO  (qtp1464304042-9063) [n:127.0.0.1:50289_solr c:replacenodetest_coll s:shard1 r:core_node3 x:replacenodetest_coll_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
   [junit4]   2> 680949 INFO  (qtp1464304042-9063) [n:127.0.0.1:50289_solr c:replacenodetest_coll s:shard1 r:core_node3 x:replacenodetest_coll_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'replacenodetest_coll_shard1_replica_n1' using configuration from collection replacenodetest_coll, trusted=true
   [junit4]   2> 680949 INFO  (qtp1464304042-9063) [n:127.0.0.1:50289_solr c:replacenodetest_coll s:shard1 r:core_node3 x:replacenodetest_coll_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50289.solr.core.replacenodetest_coll.shard1.replica_n1' (registry 'solr.core.replacenodetest_coll.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3befc08c
   [junit4]   2> 680949 INFO  (qtp1464304042-9063) [n:127.0.0.1:50289_solr c:replacenodetest_coll s:shard1 r:core_node3 x:replacenodetest_coll_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 680949 INFO  (qtp1464304042-9063) [n:127.0.0.1:50289_solr c:replacenodetest_coll s:shard1 r:core_node3 x:replacenodetest_coll_shard1_replica_n1] o.a.s.c.SolrCore [[replacenodetest_coll_shard1_replica_n1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_20D3DA769E1BC5EF-001/tempDir-001/node2/replacenodetest_coll_shard1_replica_n1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_20D3DA769E1BC5EF-001/tempDir-001/node2/./replacenodetest_coll_shard1_replica_n1/data/]
   [junit4]   2> 681088 INFO  (qtp1464304042-9064) [n:127.0.0.1:50289_solr c:replacenodetest_coll s:shard3 r:core_node13 x:replacenodetest_coll_shard3_replica_p10] o.a.s.c.SolrCore [replacenodetest_coll_shard3_replica_p10]  CLOSING SolrCore org.apache.solr.core.SolrCore@4000b7ab
   [junit4]   2> 681088 INFO  (qtp2018161192-9050) [n:127.0.0.1:50296_solr c:replacenodetest_coll s:shard3 r:core_node11 x:replacenodetest_coll_shard3_replica_n8] o.a.s.c.SolrCore [replacenodetest_coll_shard3_replica_n8]  CLOSING SolrCore org.apache.solr.core.SolrCore@2fa3dbdb
   [junit4]   2> 681088 INFO  (qtp1257807132-9076) [n:127.0.0.1:50294_solr c:replacenodetest_coll s:shard2 r:core_node7 x:replacenodetest_coll_shard2_replica_n4] o.a.s.c.SolrCore [replacenodetest_coll_shard2_replica_n4]  CLOSING SolrCore org.apache.solr.core.SolrCore@75806437
   [junit4]   2> 681088 INFO  (qtp1582697236-9054) [n:127.0.0.1:50290_solr c:replacenodetest_coll s:shard4 r:core_node15 x:replacenodetest_coll_shard4_replica_n12] o.a.s.c.SolrCore [replacenodetest_coll_shard4_replica_n12]  CLOSING SolrCore org.apache.solr.core.SolrCore@7da8e930
   [junit4]   2> 681089 INFO  (qtp1582697236-9056) [n:127.0.0.1:50290_solr c:replacenodetest_coll s:shard1 r:core_node5 x:replacenodetest_coll_shard1_replica_p2] o.a.s.c.SolrCore [replacenodetest_coll_shard1_replica_p2]  CLOSING SolrCore org.apache.solr.core.SolrCore@722cb4f6
   [junit4]   2> 681089 INFO  (qtp1792046842-9099) [n:127.0.0.1:50295_solr c:replacenodetest_coll s:shard2 r:core_node9 x:replacenodetest_coll_shard2_replica_p6] o.a.s.c.SolrCore [replacenodetest_coll_shard2_replica_p6]  CLOSING SolrCore org.apache.solr.core.SolrCore@57f07853
   [junit4]   2> 681089 INFO  (qtp1464304042-9063) [n:127.0.0.1:50289_solr c:replacenodetest_coll s:shard1 r:core_node3 x:replacenodetest_coll_shard1_replica_n1] o.a.s.c.SolrCore [replacenodetest_coll_shard1_replica_n1]  CLOSING SolrCore org.apache.solr.core.SolrCore@54bff02e
   [junit4]   2> 681089 INFO  (qtp1257807132-9073) [n:127.0.0.1:50294_solr c:replacenodetest_coll s:shard4 r:core_node17 x:replacenodetest_coll_shard4_replica_p14] o.a.s.c.SolrCore [replacenodetest_coll_shard4_replica_p14]  CLOSING SolrCore org.apache.solr.core.SolrCore@69e1524c
   [junit4]   2> 681089 INFO  (qtp2018161192-9051) [n:127.0.0.1:50296_solr c:replacenodetest_coll s:shard5 r:core_node20 x:replacenodetest_coll_shard5_replica_p18] o.a.s.c.SolrCore [replacenodetest_coll_shard5_replica_p18]  CLOSING SolrCore org.apache.solr.core.SolrCore@10cb01f6
   [junit4]   2> 681089 INFO  (qtp1464304042-9064) [n:127.0.0.1:50289_solr c:replacenodetest_coll s:shard3 r:core_node13 x:replacenodetest_coll_shard3_replica_p10] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.replacenodetest_coll.shard3.replica_p10, tag=1073788843
   [junit4]   2> 681090 INFO  (qtp2018161192-9050) [n:127.0.0.1:50296_solr c:replacenodetest_coll s:shard3 r:core_node11 x:replacenodetest_coll_shard3_replica_n8] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.replacenodetest_coll.shard3.replica_n8, tag=799267803
   [junit4]   2> 681090 INFO  (qtp1257807132-9076) [n:127.0.0.1:50294_solr c:replacenodetest_coll s:shard2 r:core_node7 x:replacenodetest_coll_shard2_replica_n4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.replacenodetest_coll.shard2.replica_n4, tag=1971348535
   [junit4]   2> 681090 INFO  (qtp1582697236-9054) [n:127.0.0.1:50290_solr c:replacenodetest_coll s:shard4 r:core_node15 x:replacenodetest_coll_shard4_replica_n12] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.replacenodetest_coll.shard4.replica_n12, tag=2108221744
   [junit4]   2> 681091 INFO  (qtp1792046842-9100) [n:127.0.0.1:50295_solr c:replacenodetest_coll s:shard5 r:core_node19 x:replacenodetest_coll_shard5_replica_n16] o.a.s.c.SolrCore [replacenodetest_coll_shard5_replica_n16]  CLOSING SolrCore org.apache.solr.core.SolrCore@9bf3411
   [junit4]   2> 681091 INFO  (qtp1792046842-9099) [n:127.0.0.1:50295_solr c:replacenodetest_coll s:shard2 r:core_node9 x:replacenodetest_coll_shard2_replica_p6] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.replacenodetest_coll.shard2.replica_p6, tag=1475377235
   [junit4]   2> 681105 INFO  (qtp1464304042-9064) [n:127.0.0.1:50289_solr c:replacenodetest_coll s:shard3 r:core_node13 x:replacenodetest_coll_shard3_replica_p10] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.replacenodetest_coll.shard3.leader, tag=1073788843
   [junit4]   2> 681105 INFO  (qtp1464304042-9063) [n:127.0.0.1:50289_solr c:replacenodetest_coll s:shard1 r:core_node3 x:replacenodetest_coll_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.replacenodetest_coll.shard1.replica_n1, tag=1421865006
   [junit4]   2> 681106 ERROR (qtp1464304042-9064) [n:127.0.0.1:50289_solr c:replacenodetest_coll s:shard3 r:core_node13 x:replacenodetest_coll_shard3_replica_p10] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: Error CREATEing SolrCore 'replacenodetest_coll_shard3_replica_p10': Unable to create core [replacenodetest_coll_shard3_replica_p10] Caused by: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:941)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
   [junit4]   2> 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:745)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:726)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:507)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:378)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:322)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:395)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: Unable to create core [replacenodetest_coll_shard3_replica_p10]
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:988)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:908)
   [junit4]   2> 	... 34 more
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:988)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:843)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:972)
   [junit4]   2> 	... 35 more
   [junit4]   2> Caused by: java.lang.RuntimeException: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.common.util.ValidatingJsonMap.parse(ValidatingJsonMap.java:318)
   [junit4]   2> 	at org.apache.solr.common.util.Utils.lambda$getSpec$0(Utils.java:394)
   [junit4]   2> 	at org.apache.solr.api.Api.getSpec(Api.java:65)
   [junit4]   2> 	at org.apache.solr.api.ApiBag.register(ApiBag.java:73)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.put(PluginBag.java:217)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.init(PluginBag.java:274)
   [junit4]   2> 	at org.apache.solr.core.RequestHandlers.initHandlersFromConfig(RequestHandlers.java:130)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:946)
   [junit4]   2> 	... 37 more
   [junit4]   2> 
   [junit4]   2> 681107 INFO  (qtp1464304042-9064) [n:127.0.0.1:50289_solr c:replacenodetest_coll s:shard3 r:core_node13 x:replacenodetest_coll_shard3_replica_p10] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node13&collection.configName=conf1&newCollection=true&name=replacenodetest_coll_shard3_replica_p10&action=CREATE&numShards=5&collection=replacenodetest_coll&shard=shard3&wt=javabin&version=2&replicaType=PULL} status=400 QTime=1352
   [junit4]   2> 681107 ERROR (OverseerThreadFactory-4276-thread-1-processing-n:127.0.0.1:50290_solr) [n:127.0.0.1:50290_solr    ] o.a.s.c.OverseerCollectionMessageHandler Error from shard: http://127.0.0.1:50289/solr
   [junit4]   2> org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:50289/solr: Error CREATEing SolrCore 'replacenodetest_coll_shard3_replica_p10': Unable to create core [replacenodetest_coll_shard3_replica_p10] Caused by: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:626)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:252)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:241)
   [junit4]   2> 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
   [junit4]   2> 	at org.apache.solr.handler.component.HttpShardHandler.lambda$submit$0(HttpShardHandler.java:172)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 681110 INFO  (qtp1257807132-9076) [n:127.0.0.1:50294_solr c:replacenodetest_coll s:shard2 r:core_node7 x:replacenodetest_coll_shard2_replica_n4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.replacenodetest_coll.shard2.leader, tag=1971348535
   [junit4]   2> 681110 INFO  (qtp1257807132-9073) [n:127.0.0.1:50294_solr c:replacenodetest_coll s:shard4 r:core_node17 x:replacenodetest_coll_shard4_replica_p14] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.replacenodetest_coll.shard4.replica_p14, tag=1776374348
   [junit4]   2> 681112 ERROR (qtp1257807132-9076) [n:127.0.0.1:50294_solr c:replacenodetest_coll s:shard2 r:core_node7 x:replacenodetest_coll_shard2_replica_n4] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: Error CREATEing SolrCore 'replacenodetest_coll_shard2_replica_n4': Unable to create core [replacenodetest_coll_shard2_replica_n4] Caused by: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:941)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
   [junit4]   2> 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:745)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:726)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:507)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:378)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:322)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:395)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: Unable to create core [replacenodetest_coll_shard2_replica_n4]
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:988)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:908)
   [junit4]   2> 	... 34 more
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:988)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:843)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:972)
   [junit4]   2> 	... 35 more
   [junit4]   2> Caused by: java.lang.RuntimeException: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.common.util.ValidatingJsonMap.parse(ValidatingJsonMap.java:318)
   [junit4]   2> 	at org.apache.solr.common.util.Utils.lambda$getSpec$0(Utils.java:394)
   [junit4]   2> 	at org.apache.solr.api.Api.getSpec(Api.java:65)
   [junit4]   2> 	at org.apache.solr.api.ApiBag.register(ApiBag.java:73)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.put(PluginBag.java:217)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.init(PluginBag.java:274)
   [junit4]   2> 	at org.apache.solr.core.RequestHandlers.initHandlersFromConfig(RequestHandlers.java:130)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:946)
   [junit4]   2> 	... 37 more
   [junit4]   2> 
   [junit4]   2> 681112 INFO  (qtp1257807132-9076) [n:127.0.0.1:50294_solr c:replacenodetest_coll s:shard2 r:core_node7 x:replacenodetest_coll_shard2_replica_n4] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node7&collection.configName=conf1&newCollection=true&name=replacenodetest_coll_shard2_replica_n4&action=CREATE&numShards=5&collection=replacenodetest_coll&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=400 QTime=1357
   [junit4]   2> 681112 ERROR (OverseerThreadFactory-4276-thread-1-processing-n:127.0.0.1:50290_solr) [n:127.0.0.1:50290_solr    ] o.a.s.c.OverseerCollectionMessageHandler Error from shard: http://127.0.0.1:50294/solr
   [junit4]   2> org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:50294/solr: Error CREATEing SolrCore 'replacenodetest_coll_shard2_replica_n4': Unable to create core [replacenodetest_coll_shard2_replica_n4] Caused by: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:626)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:252)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:241)
   [junit4]   2> 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
   [junit4]   2> 	at org.apache.solr.handler.component.HttpShardHandler.lambda$submit$0(HttpShardHandler.java:172)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Wo

[...truncated too long message...]

cj-lint] ----------
 [ecj-lint] 8. WARNING in /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java (at line 578)
 [ecj-lint] 	tvWindowStream = new OffsetWindowTokenFilter(tvStream);
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'tvWindowStream' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 9. WARNING in /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/java/org/apache/solr/request/SimpleFacets.java (at line 912)
 [ecj-lint] 	fastForRandomSet = new HashDocSet(sset.getDocs(), 0, sset.size());
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'fastForRandomSet' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 10. WARNING in /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/java/org/apache/solr/response/SmileResponseWriter.java (at line 33)
 [ecj-lint] 	new SmileWriter(out, request, response).writeResponse();
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: '<unassigned Closeable value>' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 11. WARNING in /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/java/org/apache/solr/schema/OpenExchangeRatesOrgProvider.java (at line 146)
 [ecj-lint] 	ratesJsonStream = resourceLoader.openResource(ratesFileLocation);
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'ratesJsonStream' is not closed at this location
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 12. WARNING in /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java (at line 1315)
 [ecj-lint] 	UninvertingReader uninvertingReader = new UninvertingReader(new ReaderWrapper(searcher.getSlowAtomicReader(), collapseField), mapping);
 [ecj-lint] 	                  ^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'uninvertingReader' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 13. WARNING in /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/java/org/apache/solr/search/DocSlice.java (at line 145)
 [ecj-lint] 	HashDocSet h = new HashDocSet(docs,offset,len);
 [ecj-lint] 	           ^
 [ecj-lint] Resource leak: 'h' is never closed
 [ecj-lint] ----------
 [ecj-lint] 14. WARNING in /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/java/org/apache/solr/search/DocSlice.java (at line 154)
 [ecj-lint] 	HashDocSet h = new HashDocSet(docs,offset,len);
 [ecj-lint] 	           ^
 [ecj-lint] Resource leak: 'h' is never closed
 [ecj-lint] ----------
 [ecj-lint] 15. WARNING in /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/java/org/apache/solr/search/DocSlice.java (at line 163)
 [ecj-lint] 	HashDocSet h = new HashDocSet(docs,offset,len);
 [ecj-lint] 	           ^
 [ecj-lint] Resource leak: 'h' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 16. WARNING in /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/java/org/apache/solr/search/function/OrdFieldSource.java (at line 78)
 [ecj-lint] 	SolrIndexSearcher is = (SolrIndexSearcher) o;
 [ecj-lint] 	                  ^^
 [ecj-lint] Resource leak: 'is' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 17. WARNING in /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/java/org/apache/solr/search/function/ReverseOrdFieldSource.java (at line 78)
 [ecj-lint] 	SolrIndexSearcher is = (SolrIndexSearcher) o;
 [ecj-lint] 	                  ^^
 [ecj-lint] Resource leak: 'is' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 18. WARNING in /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java (at line 287)
 [ecj-lint] 	throw new SolrException(ErrorCode.SERVER_ERROR, "Error occurred while loading solr.xml from zookeeper", e);
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'loader' is not closed at this location
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 19. WARNING in /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/java/org/apache/solr/update/CdcrTransactionLog.java (at line 98)
 [ecj-lint] 	ChannelFastInputStream is = new ChannelFastInputStream(channel, pos);
 [ecj-lint] 	                       ^^
 [ecj-lint] Resource leak: 'is' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 20. WARNING in /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/java/org/apache/solr/update/HdfsTransactionLog.java (at line 191)
 [ecj-lint] 	LogCodec codec = new LogCodec(resolver);
 [ecj-lint] 	         ^^^^^
 [ecj-lint] Resource leak: 'codec' is never closed
 [ecj-lint] ----------
 [ecj-lint] 21. WARNING in /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/java/org/apache/solr/update/HdfsTransactionLog.java (at line 261)
 [ecj-lint] 	LogCodec codec = new LogCodec(resolver);
 [ecj-lint] 	         ^^^^^
 [ecj-lint] Resource leak: 'codec' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 22. WARNING in /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/java/org/apache/solr/update/TransactionLog.java (at line 228)
 [ecj-lint] 	ChannelFastInputStream is = new ChannelFastInputStream(channel, pos);
 [ecj-lint] 	                       ^^
 [ecj-lint] Resource leak: 'is' is never closed
 [ecj-lint] ----------
 [ecj-lint] 23. WARNING in /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/java/org/apache/solr/update/TransactionLog.java (at line 228)
 [ecj-lint] 	ChannelFastInputStream is = new ChannelFastInputStream(channel, pos);
 [ecj-lint] 	                       ^^
 [ecj-lint] Resource leak: 'is' is never closed
 [ecj-lint] ----------
 [ecj-lint] 24. WARNING in /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/java/org/apache/solr/update/TransactionLog.java (at line 260)
 [ecj-lint] 	LogCodec codec = new LogCodec(resolver);
 [ecj-lint] 	         ^^^^^
 [ecj-lint] Resource leak: 'codec' is never closed
 [ecj-lint] ----------
 [ecj-lint] 25. WARNING in /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/java/org/apache/solr/update/TransactionLog.java (at line 275)
 [ecj-lint] 	LogCodec codec = new LogCodec(resolver);
 [ecj-lint] 	         ^^^^^
 [ecj-lint] Resource leak: 'codec' is never closed
 [ecj-lint] ----------
 [ecj-lint] 26. WARNING in /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/java/org/apache/solr/update/TransactionLog.java (at line 529)
 [ecj-lint] 	LogCodec codec = new LogCodec(resolver);
 [ecj-lint] 	         ^^^^^
 [ecj-lint] Resource leak: 'codec' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 27. ERROR in /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/java/org/apache/solr/update/processor/UUIDUpdateProcessorFactory.java (at line 23)
 [ecj-lint] 	import org.apache.solr.common.SolrException;
 [ecj-lint] 	       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] The import org.apache.solr.common.SolrException is never used
 [ecj-lint] ----------
 [ecj-lint] 28. ERROR in /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/java/org/apache/solr/update/processor/UUIDUpdateProcessorFactory.java (at line 24)
 [ecj-lint] 	import static org.apache.solr.common.SolrException.ErrorCode.*;
 [ecj-lint] 	              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] The import org.apache.solr.common.SolrException.ErrorCode is never used
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 29. WARNING in /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessor.java (at line 85)
 [ecj-lint] 	UpdateRequestProcessor p = this;
 [ecj-lint] 	                       ^
 [ecj-lint] Resource leak: 'p' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 30. WARNING in /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/java/org/apache/solr/util/SimplePostTool.java (at line 655)
 [ecj-lint] 	BAOS bos =  new BAOS();
 [ecj-lint] 	     ^^^
 [ecj-lint] Resource leak: 'bos' is never closed
 [ecj-lint] ----------
 [ecj-lint] 31. WARNING in /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/java/org/apache/solr/util/SimplePostTool.java (at line 655)
 [ecj-lint] 	BAOS bos =  new BAOS();
 [ecj-lint] 	     ^^^
 [ecj-lint] Resource leak: 'bos' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 32. WARNING in /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/java/org/apache/solr/util/SolrLogLayout.java (at line 149)
 [ecj-lint] 	SolrQueryRequest req = requestInfo == null ? null : requestInfo.getReq();
 [ecj-lint] 	                 ^^^
 [ecj-lint] Resource leak: 'req' is never closed
 [ecj-lint] ----------
 [ecj-lint] 32 problems (2 errors, 30 warnings)

BUILD FAILED
/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/build.xml:810: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/build.xml:101: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build.xml:689: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/lucene/common-build.xml:2007: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/lucene/common-build.xml:2046: Compile failed; see the compiler error output for details.

Total time: 90 minutes 18 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any

[JENKINS] Lucene-Solr-7.x-MacOSX (64bit/jdk1.8.0) - Build # 82 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-MacOSX/82/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseSerialGC

6 tests failed.
FAILED:  org.apache.solr.cloud.CustomCollectionTest.testCreateShardRepFactor

Error Message:
Error from server at http://127.0.0.1:58202/solr: Could not find collection : testCreateShardRepFactor

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:58202/solr: Could not find collection : testCreateShardRepFactor
	at __randomizedtesting.SeedInfo.seed([BAC0A1C0CDC51896:4E2E2B44D5910D56]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:626)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:252)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:241)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1121)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:862)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
	at org.apache.solr.cloud.CustomCollectionTest.testCreateShardRepFactor(CustomCollectionTest.java:190)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.DistribDocExpirationUpdateProcessorTest

Error Message:


Stack Trace:
java.util.concurrent.TimeoutException
	at __randomizedtesting.SeedInfo.seed([BAC0A1C0CDC51896]:0)
	at org.apache.solr.common.cloud.ZkStateReader.waitForState(ZkStateReader.java:1266)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.waitForState(CloudSolrClient.java:438)
	at org.apache.solr.cloud.DistribDocExpirationUpdateProcessorTest.setupCluster(DistribDocExpirationUpdateProcessorTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:847)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.MigrateRouteKeyTest.multipleShardMigrateTest

Error Message:
Collection not found: sourceCollection

Stack Trace:
org.apache.solr.common.SolrException: Collection not found: sourceCollection
	at __randomizedtesting.SeedInfo.seed([BAC0A1C0CDC51896:A6D0BBBAEC05A877]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.getCollectionNames(CloudSolrClient.java:1139)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:822)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:173)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:138)
	at org.apache.solr.cloud.MigrateRouteKeyTest.multipleShardMigrateTest(MigrateRouteKeyTest.java:104)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.MultiThreadedOCPTest.test

Error Message:


Stack Trace:
java.lang.NullPointerException
	at __randomizedtesting.SeedInfo.seed([BAC0A1C0CDC51896:32949E1A6339756E]:0)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.getNodeName(JettySolrRunner.java:345)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:415)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:334)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:983)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.MultiThreadedOCPTest

Error Message:
42 threads leaked from SUITE scope at org.apache.solr.cloud.MultiThreadedOCPTest:     1) Thread[id=17662, name=qtp2034303826-17662, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=17716, name=zkCallback-2743-thread-3, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=17666, name=qtp2034303826-17666, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=17697, name=qtp1930997315-17697, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=17675, name=Connection evictor, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=17704, name=Connection evictor, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)    7) Thread[id=17667, name=qtp2034303826-17667, state=RUNNABLE, group=TGRP-MultiThreadedOCPTest]         at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)         at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)         at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:748)    8) Thread[id=17665, name=qtp2034303826-17665, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)    9) Thread[id=17681, name=zkCallback-2743-thread-1-processing-n:127.0.0.1:58775_cp%2Fbv, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:524)         at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:467)         at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:454)         at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:763)         at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)         at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)         at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:56)         at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:348)         at org.apache.solr.common.cloud.SolrZkClient$3.lambda$process$0(SolrZkClient.java:269)         at org.apache.solr.common.cloud.SolrZkClient$3$$Lambda$233/1833355619.run(Unknown Source)         at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)         at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$89/385221798.run(Unknown Source)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   10) Thread[id=17670, name=org.eclipse.jetty.server.session.HashSessionManager@6b9facbaTimer, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   11) Thread[id=17693, name=qtp1930997315-17693, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   12) Thread[id=17690, name=Connection evictor, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)   13) Thread[id=17684, name=Scheduler-265837956, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   14) Thread[id=17691, name=qtp1930997315-17691, state=RUNNABLE, group=TGRP-MultiThreadedOCPTest]         at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)         at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)         at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:748)   15) Thread[id=17699, name=org.eclipse.jetty.server.session.HashSessionManager@3d4f72ceTimer, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   16) Thread[id=17703, name=Connection evictor, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)   17) Thread[id=17715, name=zkCallback-2743-thread-2, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   18) Thread[id=17682, name=Thread-3559, state=WAITING, group=TGRP-MultiThreadedOCPTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1591)   19) Thread[id=17720, name=zkCallback-2749-thread-2, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   20) Thread[id=17663, name=qtp2034303826-17663-acceptor-0@7136ddb3-ServerConnector@7010b6f4{HTTP/1.1,[http/1.1]}{127.0.0.1:58775}, state=RUNNABLE, group=TGRP-MultiThreadedOCPTest]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:748)   21) Thread[id=17694, name=qtp1930997315-17694, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   22) Thread[id=17674, name=Connection evictor, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)   23) Thread[id=17695, name=qtp1930997315-17695, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   24) Thread[id=17710, name=zkCallback-2749-thread-1, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   25) Thread[id=17708, name=TEST-MultiThreadedOCPTest.test-seed#[BAC0A1C0CDC51896]-SendThread(127.0.0.1:58757), state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)   26) Thread[id=17717, name=zkCallback-2743-thread-4, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   27) Thread[id=17669, name=qtp2034303826-17669, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   28) Thread[id=17683, name=coreContainerWorkExecutor-9491-thread-1, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   29) Thread[id=17719, name=zkCallback-2743-thread-6, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   30) Thread[id=17680, name=TEST-MultiThreadedOCPTest.test-seed#[BAC0A1C0CDC51896]-EventThread, state=WAITING, group=TGRP-MultiThreadedOCPTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)   31) Thread[id=17692, name=qtp1930997315-17692-acceptor-0@321ac6f0-ServerConnector@260b5e9c{HTTP/1.1,[http/1.1]}{127.0.0.1:58782}, state=RUNNABLE, group=TGRP-MultiThreadedOCPTest]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:748)   32) Thread[id=17709, name=TEST-MultiThreadedOCPTest.test-seed#[BAC0A1C0CDC51896]-EventThread, state=WAITING, group=TGRP-MultiThreadedOCPTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)   33) Thread[id=17718, name=zkCallback-2743-thread-5, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   34) Thread[id=17721, name=zkCallback-2749-thread-3, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   35) Thread[id=17686, name=watches-2744-thread-1, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   36) Thread[id=17722, name=zkCallback-2749-thread-4, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   37) Thread[id=17698, name=qtp1930997315-17698, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   38) Thread[id=17664, name=qtp2034303826-17664, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   39) Thread[id=17679, name=TEST-MultiThreadedOCPTest.test-seed#[BAC0A1C0CDC51896]-SendThread(127.0.0.1:58757), state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)   40) Thread[id=17668, name=qtp2034303826-17668, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   41) Thread[id=17688, name=searcherExecutor-9496-thread-1, state=WAITING, group=TGRP-MultiThreadedOCPTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   42) Thread[id=17696, name=qtp1930997315-17696, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 42 threads leaked from SUITE scope at org.apache.solr.cloud.MultiThreadedOCPTest: 
   1) Thread[id=17662, name=qtp2034303826-17662, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=17716, name=zkCallback-2743-thread-3, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=17666, name=qtp2034303826-17666, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=17697, name=qtp1930997315-17697, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=17675, name=Connection evictor, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=17704, name=Connection evictor, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
   7) Thread[id=17667, name=qtp2034303826-17667, state=RUNNABLE, group=TGRP-MultiThreadedOCPTest]
        at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
        at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
        at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:748)
   8) Thread[id=17665, name=qtp2034303826-17665, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
   9) Thread[id=17681, name=zkCallback-2743-thread-1-processing-n:127.0.0.1:58775_cp%2Fbv, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:524)
        at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:467)
        at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:454)
        at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:763)
        at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)
        at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)
        at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:56)
        at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:348)
        at org.apache.solr.common.cloud.SolrZkClient$3.lambda$process$0(SolrZkClient.java:269)
        at org.apache.solr.common.cloud.SolrZkClient$3$$Lambda$233/1833355619.run(Unknown Source)
        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$89/385221798.run(Unknown Source)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  10) Thread[id=17670, name=org.eclipse.jetty.server.session.HashSessionManager@6b9facbaTimer, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  11) Thread[id=17693, name=qtp1930997315-17693, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  12) Thread[id=17690, name=Connection evictor, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
  13) Thread[id=17684, name=Scheduler-265837956, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  14) Thread[id=17691, name=qtp1930997315-17691, state=RUNNABLE, group=TGRP-MultiThreadedOCPTest]
        at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
        at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
        at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
        at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:748)
  15) Thread[id=17699, name=org.eclipse.jetty.server.session.HashSessionManager@3d4f72ceTimer, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  16) Thread[id=17703, name=Connection evictor, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
  17) Thread[id=17715, name=zkCallback-2743-thread-2, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  18) Thread[id=17682, name=Thread-3559, state=WAITING, group=TGRP-MultiThreadedOCPTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1591)
  19) Thread[id=17720, name=zkCallback-2749-thread-2, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  20) Thread[id=17663, name=qtp2034303826-17663-acceptor-0@7136ddb3-ServerConnector@7010b6f4{HTTP/1.1,[http/1.1]}{127.0.0.1:58775}, state=RUNNABLE, group=TGRP-MultiThreadedOCPTest]
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
        at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373)
        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:748)
  21) Thread[id=17694, name=qtp1930997315-17694, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  22) Thread[id=17674, name=Connection evictor, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
  23) Thread[id=17695, name=qtp1930997315-17695, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  24) Thread[id=17710, name=zkCallback-2749-thread-1, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  25) Thread[id=17708, name=TEST-MultiThreadedOCPTest.test-seed#[BAC0A1C0CDC51896]-SendThread(127.0.0.1:58757), state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
  26) Thread[id=17717, name=zkCallback-2743-thread-4, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  27) Thread[id=17669, name=qtp2034303826-17669, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  28) Thread[id=17683, name=coreContainerWorkExecutor-9491-thread-1, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  29) Thread[id=17719, name=zkCallback-2743-thread-6, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  30) Thread[id=17680, name=TEST-MultiThreadedOCPTest.test-seed#[BAC0A1C0CDC51896]-EventThread, state=WAITING, group=TGRP-MultiThreadedOCPTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
  31) Thread[id=17692, name=qtp1930997315-17692-acceptor-0@321ac6f0-ServerConnector@260b5e9c{HTTP/1.1,[http/1.1]}{127.0.0.1:58782}, state=RUNNABLE, group=TGRP-MultiThreadedOCPTest]
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
        at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373)
        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:748)
  32) Thread[id=17709, name=TEST-MultiThreadedOCPTest.test-seed#[BAC0A1C0CDC51896]-EventThread, state=WAITING, group=TGRP-MultiThreadedOCPTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
  33) Thread[id=17718, name=zkCallback-2743-thread-5, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  34) Thread[id=17721, name=zkCallback-2749-thread-3, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  35) Thread[id=17686, name=watches-2744-thread-1, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  36) Thread[id=17722, name=zkCallback-2749-thread-4, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  37) Thread[id=17698, name=qtp1930997315-17698, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  38) Thread[id=17664, name=qtp2034303826-17664, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  39) Thread[id=17679, name=TEST-MultiThreadedOCPTest.test-seed#[BAC0A1C0CDC51896]-SendThread(127.0.0.1:58757), state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
  40) Thread[id=17668, name=qtp2034303826-17668, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  41) Thread[id=17688, name=searcherExecutor-9496-thread-1, state=WAITING, group=TGRP-MultiThreadedOCPTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  42) Thread[id=17696, name=qtp1930997315-17696, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([BAC0A1C0CDC51896]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.MultiThreadedOCPTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=17716, name=zkCallback-2743-thread-3, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=17682, name=Thread-3559, state=WAITING, group=TGRP-MultiThreadedOCPTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1591)    3) Thread[id=17720, name=zkCallback-2749-thread-2, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=17710, name=zkCallback-2749-thread-1, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=17708, name=TEST-MultiThreadedOCPTest.test-seed#[BAC0A1C0CDC51896]-SendThread(127.0.0.1:58757), state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)    6) Thread[id=17717, name=zkCallback-2743-thread-4, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    7) Thread[id=17667, name=qtp2034303826-17667, state=RUNNABLE, group=TGRP-MultiThreadedOCPTest]         at sun.nio.ch.KQueueArrayWrapper.interrupt(Native Method)         at sun.nio.ch.KQueueArrayWrapper.interrupt(KQueueArrayWrapper.java:203)         at sun.nio.ch.KQueueSelectorImpl.wakeup(KQueueSelectorImpl.java:257)         at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)         at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)         at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:116)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:748)    8) Thread[id=17683, name=coreContainerWorkExecutor-9491-thread-1, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    9) Thread[id=17719, name=zkCallback-2743-thread-6, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   10) Thread[id=17681, name=zkCallback-2743-thread-1, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   11) Thread[id=17670, name=org.eclipse.jetty.server.session.HashSessionManager@6b9facbaTimer, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecu

[...truncated too long message...]

olExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   13) Thread[id=17684, name=Scheduler-265837956, state=WAITING, group=TGRP-MultiThreadedOCPTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   14) Thread[id=17721, name=zkCallback-2749-thread-3, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   15) Thread[id=17686, name=watches-2744-thread-1, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   16) Thread[id=17722, name=zkCallback-2749-thread-4, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   17) Thread[id=17691, name=qtp1930997315-17691, state=RUNNABLE, group=TGRP-MultiThreadedOCPTest]
   [junit4]    >         at sun.nio.ch.KQueueArrayWrapper.interrupt(Native Method)
   [junit4]    >         at sun.nio.ch.KQueueArrayWrapper.interrupt(KQueueArrayWrapper.java:203)
   [junit4]    >         at sun.nio.ch.KQueueSelectorImpl.wakeup(KQueueSelectorImpl.java:257)
   [junit4]    >         at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
   [junit4]    >         at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
   [junit4]    >         at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:116)
   [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   18) Thread[id=17699, name=org.eclipse.jetty.server.session.HashSessionManager@3d4f72ceTimer, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   19) Thread[id=17679, name=TEST-MultiThreadedOCPTest.test-seed#[BAC0A1C0CDC51896]-SendThread(127.0.0.1:58757), state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
   [junit4]    >   20) Thread[id=17715, name=zkCallback-2743-thread-2, state=TIMED_WAITING, group=TGRP-MultiThreadedOCPTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   21) Thread[id=17688, name=searcherExecutor-9496-thread-1, state=WAITING, group=TGRP-MultiThreadedOCPTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([BAC0A1C0CDC51896]:0)
   [junit4] Completed [531/726 (4!)] on J1 in 30.91s, 1 test, 3 errors <<< FAILURES!

[...truncated 43988 lines...]

[JENKINS] Lucene-Solr-7.x-MacOSX (64bit/jdk1.8.0) - Build # 81 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-MacOSX/81/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseSerialGC

12 tests failed.
FAILED:  org.apache.solr.cloud.CustomCollectionTest.testRouteFieldForHashRouter

Error Message:
Collection not found: routeFieldColl

Stack Trace:
org.apache.solr.common.SolrException: Collection not found: routeFieldColl
	at __randomizedtesting.SeedInfo.seed([7D00A8B605D57C02:D536366B9AB49758]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.getCollectionNames(CloudSolrClient.java:1139)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:822)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
	at org.apache.solr.client.solrj.request.UpdateRequest.commit(UpdateRequest.java:233)
	at org.apache.solr.cloud.CustomCollectionTest.testRouteFieldForHashRouter(CustomCollectionTest.java:166)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.CustomCollectionTest.testCustomCollectionsAPI

Error Message:
Could not find collection : implicitcoll

Stack Trace:
org.apache.solr.common.SolrException: Could not find collection : implicitcoll
	at __randomizedtesting.SeedInfo.seed([7D00A8B605D57C02:17E126DD384FCA7A]:0)
	at org.apache.solr.common.cloud.ClusterState.getCollection(ClusterState.java:194)
	at org.apache.solr.cloud.SolrCloudTestCase.getCollectionState(SolrCloudTestCase.java:247)
	at org.apache.solr.cloud.CustomCollectionTest.testCustomCollectionsAPI(CustomCollectionTest.java:68)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.ShardSplitTest.test

Error Message:
Could not find new collection routeFieldColl

Stack Trace:
java.lang.AssertionError: Could not find new collection routeFieldColl
	at __randomizedtesting.SeedInfo.seed([7D00A8B605D57C02:F554976CAB2911FA]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkForCollection(AbstractFullDistribZkTestBase.java:1813)
	at org.apache.solr.cloud.ShardSplitTest.splitByRouteFieldTest(ShardSplitTest.java:689)
	at org.apache.solr.cloud.ShardSplitTest.test(ShardSplitTest.java:101)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.ShardSplitTest.testSplitStaticIndexReplication

Error Message:
Error from server at http://127.0.0.1:61942: ADDREPLICA failed to create replica

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:61942: ADDREPLICA failed to create replica
	at __randomizedtesting.SeedInfo.seed([7D00A8B605D57C02:374A3CB6947D5307]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:626)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:252)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:241)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1121)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:862)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:408)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:334)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:983)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.ShardSplitTest.testSplitAfterFailedSplit

Error Message:


Stack Trace:
java.lang.NullPointerException
	at __randomizedtesting.SeedInfo.seed([7D00A8B605D57C02:844D3B1939A03188]:0)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.getNodeName(JettySolrRunner.java:345)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:314)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:983)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.ShardSplitTest.testSplitShardWithRule

Error Message:
java.io.IOException: Couldn't instantiate org.apache.zookeeper.ClientCnxnSocketNIO

Stack Trace:
org.apache.solr.common.SolrException: java.io.IOException: Couldn't instantiate org.apache.zookeeper.ClientCnxnSocketNIO
	at __randomizedtesting.SeedInfo.seed([7D00A8B605D57C02:FC0E790498272729]:0)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:171)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:117)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:112)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:99)
	at org.apache.solr.cloud.AbstractZkTestCase.buildZooKeeper(AbstractZkTestCase.java:92)
	at org.apache.solr.cloud.AbstractZkTestCase.buildZooKeeper(AbstractZkTestCase.java:82)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.distribSetUp(AbstractDistribZkTestBase.java:81)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.distribSetUp(AbstractFullDistribZkTestBase.java:225)
	at org.apache.solr.cloud.ShardSplitTest.distribSetUp(ShardSplitTest.java:85)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:955)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.io.IOException: Couldn't instantiate org.apache.zookeeper.ClientCnxnSocketNIO
	at org.apache.zookeeper.ZooKeeper.getClientCnxnSocket(ZooKeeper.java:1842)
	at org.apache.zookeeper.ZooKeeper.<init>(ZooKeeper.java:447)
	at org.apache.zookeeper.ZooKeeper.<init>(ZooKeeper.java:380)
	at org.apache.solr.common.cloud.SolrZooKeeper.<init>(SolrZooKeeper.java:43)
	at org.apache.solr.common.cloud.ZkClientConnectionStrategy.createSolrZooKeeper(ZkClientConnectionStrategy.java:105)
	at org.apache.solr.common.cloud.DefaultConnectionStrategy.connect(DefaultConnectionStrategy.java:37)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:146)
	... 40 more
Caused by: java.io.IOException: Too many open files
	at sun.nio.ch.IOUtil.makePipe(Native Method)
	at sun.nio.ch.KQueueSelectorImpl.<init>(KQueueSelectorImpl.java:84)
	at sun.nio.ch.KQueueSelectorProvider.openSelector(KQueueSelectorProvider.java:42)
	at java.nio.channels.Selector.open(Selector.java:227)
	at org.apache.zookeeper.ClientCnxnSocketNIO.<init>(ClientCnxnSocketNIO.java:43)
	at sun.reflect.GeneratedConstructorAccessor25.newInstance(Unknown Source)
	at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
	at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
	at java.lang.Class.newInstance(Class.java:442)
	at org.apache.zookeeper.ZooKeeper.getClientCnxnSocket(ZooKeeper.java:1839)
	... 46 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ShardSplitTest

Error Message:
39 threads leaked from SUITE scope at org.apache.solr.cloud.ShardSplitTest:     1) Thread[id=20226, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-ShardSplitTest]         at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)         at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)         at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:173)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=20209, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=20168, name=TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[7D00A8B605D57C02]-EventThread, state=WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)    4) Thread[id=20182, name=zkCallback-3074-thread-3, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=20225, name=Thread-3727, state=WAITING, group=TGRP-ShardSplitTest]         at java.lang.Object.wait(Native Method)         at java.lang.Thread.join(Thread.java:1252)         at java.lang.Thread.join(Thread.java:1326)         at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:297)         at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:309)         at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:490)    6) Thread[id=20175, name=watches-3075-thread-1, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    7) Thread[id=20221, name=zkCallback-3082-thread-2, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    8) Thread[id=20222, name=zkCallback-3082-thread-3, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    9) Thread[id=20219, name=OverseerHdfsCoreFailoverThread-98386508698615812-127.0.0.1:61953_-n_0000000000, state=WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1406)         at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1210)         at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:357)         at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:354)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)         at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1124)         at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1110)         at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:646)         at org.apache.solr.common.cloud.ClusterState.getCollectionsMap(ClusterState.java:250)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.doWork(OverseerAutoReplicaFailoverThread.java:166)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:131)         at java.lang.Thread.run(Thread.java:748)   10) Thread[id=20169, name=zkCallback-3074-thread-1, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   11) Thread[id=20171, name=Thread-3717, state=WAITING, group=TGRP-ShardSplitTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1591)   12) Thread[id=20153, name=qtp994364532-20153, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   13) Thread[id=20154, name=qtp994364532-20154, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   14) Thread[id=20217, name=OverseerStateUpdate-98386508698615812-127.0.0.1:61953_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:165)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:139)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:184)         at java.lang.Thread.run(Thread.java:748)   15) Thread[id=20214, name=TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[7D00A8B605D57C02]-SendThread(127.0.0.1:61950), state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)   16) Thread[id=20167, name=TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[7D00A8B605D57C02]-SendThread(127.0.0.1:61913), state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)   17) Thread[id=20150, name=qtp994364532-20150, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   18) Thread[id=20184, name=zkCallback-3074-thread-5, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   19) Thread[id=20210, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)   20) Thread[id=20152, name=qtp994364532-20152, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   21) Thread[id=20163, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)   22) Thread[id=20172, name=coreContainerWorkExecutor-9076-thread-1, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   23) Thread[id=20151, name=qtp994364532-20151-acceptor-0@2a810117-ServerConnector@2d94e4fd{HTTP/1.1,[http/1.1]}{127.0.0.1:61942}, state=RUNNABLE, group=TGRP-ShardSplitTest]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:748)   24) Thread[id=20155, name=qtp994364532-20155, state=RUNNABLE, group=TGRP-ShardSplitTest]         at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)         at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)         at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:748)   25) Thread[id=20156, name=qtp994364532-20156, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   26) Thread[id=20229, name=ProcessThread(sid:0 cport:61959):, state=WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:122)   27) Thread[id=20185, name=zkCallback-3074-thread-6, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   28) Thread[id=20218, name=OverseerCollectionConfigSetProcessor-98386508698615812-127.0.0.1:61953_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)         at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:387)         at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:193)         at java.lang.Thread.run(Thread.java:748)   29) Thread[id=20157, name=qtp994364532-20157, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   30) Thread[id=20183, name=zkCallback-3074-thread-4, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   31) Thread[id=20223, name=zkCallback-3082-thread-4, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   32) Thread[id=20162, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)   33) Thread[id=20181, name=zkCallback-3074-thread-2-processing-n:127.0.0.1:61942_, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:303)         at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:551)         at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:467)         at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:454)         at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:763)         at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)         at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)         at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:56)         at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:348)         at org.apache.solr.common.cloud.SolrZkClient$3.lambda$process$0(SolrZkClient.java:269)         at org.apache.solr.common.cloud.SolrZkClient$3$$Lambda$9/1716813974.run(Unknown Source)         at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)         at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$10/2018257710.run(Unknown Source)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   34) Thread[id=20224, name=zkCallback-3082-thread-5, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   35) Thread[id=20173, name=Scheduler-1600548813, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   36) Thread[id=20215, name=TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[7D00A8B605D57C02]-EventThread, state=WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)   37) Thread[id=20227, name=SessionTracker, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at java.lang.Object.wait(Native Method)         at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146)   38) Thread[id=20158, name=org.eclipse.jetty.server.session.HashSessionManager@1f938448Timer, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   39) Thread[id=20216, name=zkCallback-3082-thread-1, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 39 threads leaked from SUITE scope at org.apache.solr.cloud.ShardSplitTest: 
   1) Thread[id=20226, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-ShardSplitTest]
        at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
        at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
        at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:173)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=20209, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=20168, name=TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[7D00A8B605D57C02]-EventThread, state=WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
   4) Thread[id=20182, name=zkCallback-3074-thread-3, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=20225, name=Thread-3727, state=WAITING, group=TGRP-ShardSplitTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Thread.join(Thread.java:1252)
        at java.lang.Thread.join(Thread.java:1326)
        at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:297)
        at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:309)
        at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:490)
   6) Thread[id=20175, name=watches-3075-thread-1, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   7) Thread[id=20221, name=zkCallback-3082-thread-2, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   8) Thread[id=20222, name=zkCallback-3082-thread-3, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   9) Thread[id=20219, name=OverseerHdfsCoreFailoverThread-98386508698615812-127.0.0.1:61953_-n_0000000000, state=WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1406)
        at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1210)
        at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:357)
        at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:354)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)
        at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1124)
        at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1110)
        at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:646)
        at org.apache.solr.common.cloud.ClusterState.getCollectionsMap(ClusterState.java:250)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.doWork(OverseerAutoReplicaFailoverThread.java:166)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:131)
        at java.lang.Thread.run(Thread.java:748)
  10) Thread[id=20169, name=zkCallback-3074-thread-1, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  11) Thread[id=20171, name=Thread-3717, state=WAITING, group=TGRP-ShardSplitTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1591)
  12) Thread[id=20153, name=qtp994364532-20153, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  13) Thread[id=20154, name=qtp994364532-20154, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  14) Thread[id=20217, name=OverseerStateUpdate-98386508698615812-127.0.0.1:61953_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:165)
        at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:139)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:184)
        at java.lang.Thread.run(Thread.java:748)
  15) Thread[id=20214, name=TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[7D00A8B605D57C02]-SendThread(127.0.0.1:61950), state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
  16) Thread[id=20167, name=TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[7D00A8B605D57C02]-SendThread(127.0.0.1:61913), state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
  17) Thread[id=20150, name=qtp994364532-20150, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  18) Thread[id=20184, name=zkCallback-3074-thread-5, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  19) Thread[id=20210, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
  20) Thread[id=20152, name=qtp994364532-20152, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  21) Thread[id=20163, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
  22) Thread[id=20172, name=coreContainerWorkExecutor-9076-thread-1, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  23) Thread[id=20151, name=qtp994364532-20151-acceptor-0@2a810117-ServerConnector@2d94e4fd{HTTP/1.1,[http/1.1]}{127.0.0.1:61942}, state=RUNNABLE, group=TGRP-ShardSplitTest]
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
        at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373)
        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:748)
  24) Thread[id=20155, name=qtp994364532-20155, state=RUNNABLE, group=TGRP-ShardSplitTest]
        at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
        at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
        at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:748)
  25) Thread[id=20156, name=qtp994364532-20156, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  26) Thread[id=20229, name=ProcessThread(sid:0 cport:61959):, state=WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:122)
  27) Thread[id=20185, name=zkCallback-3074-thread-6, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  28) Thread[id=20218, name=OverseerCollectionConfigSetProcessor-98386508698615812-127.0.0.1:61953_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)
        at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:387)
        at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:193)
        at java.lang.Thread.run(Thread.java:748)
  29) Thread[id=20157, name=qtp994364532-20157, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  30) Thread[id=20183, name=zkCallback-3074-thread-4, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  31) Thread[id=20223, name=zkCallback-3082-thread-4, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  32) Thread[id=20162, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
  33) Thread[id=20181, name=zkCallback-3074-thread-2-processing-n:127.0.0.1:61942_, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:303)
        at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:551)
        at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:467)
        at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:454)
        at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:763)
        at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)
        at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)
        at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:56)
        at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:348)
        at org.apache.solr.common.cloud.SolrZkClient$3.lambda$process$0(SolrZkClient.java:269)
        at org.apache.solr.common.cloud.SolrZkClient$3$$Lambda$9/1716813974.run(Unknown Source)
        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$10/2018257710.run(Unknown Source)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  34) Thread[id=20224, name=zkCallback-3082-thread-5, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  35) Thread[id=20173, name=Scheduler-1600548813, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  36) Thread[id=20215, name=TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[7D00A8B605D57C02]-EventThread, state=WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
  37) Thread[id=20227, name=SessionTracker, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at java.lang.Object.wait(Native Method)
        at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146)
  38) Thread[id=20158, name=org.eclipse.jetty.server.session.HashSessionManager@1f938448Timer, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  39) Thread[id=20216, name=zkCallback-3082-thread-1, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([7D00A8B605D57C02]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ShardSplitTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=20226, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-ShardSplitTest]         at sun.nio.ch.KQueueArrayWrapper.interrupt(Native Method)         at sun.nio.ch.KQueueArrayWrapper.interrupt(KQueueArrayWrapper.java:203)         at sun.nio.ch.KQueueSelectorImpl.wakeup(KQueueSelectorImpl.java:257)         at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)         at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)         at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:116)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:173)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=20214, name=TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[7D00A8B605D57C02]-SendThread(127.0.0.1:61950), state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)    3) Thread[id=20167, name=TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[7D00A8B605D57C02]-SendThread(127.0.0.1:61913), state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)    4) Thread[id=20184, name=zkCallback-3074-thread-5, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=20172, name=coreContainerWorkExecutor-9076-thread-1, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=20155, name=qtp994364532-20155, state=RUNNABLE, group=TGRP-ShardSplitTest]         at sun.nio.ch.KQueueArrayWrapper.interrupt(Native Method)         at sun

[...truncated too long message...]

/tlog_replica_test_out_of_order_db_qwith_in_place_updates/state.json
   [junit4]   2> 	13	/solr/collections/tlog_replica_test_recovery/state.json
   [junit4]   2> 	13	/solr/collections/tlog_replica_test_basic_leader_election/state.json
   [junit4]   2> 	13	/solr/collections/tlog_replica_test_only_leader_indexes/state.json
   [junit4]   2> 	13	/solr/collections/tlog_replica_test_delete_by_id/state.json
   [junit4]   2> 	6	/solr/collections/tlog_replica_test_add_docs/state.json
   [junit4]   2> 	6	/solr/collections/tlog_replica_test_real_time_get/state.json
   [junit4]   2> 	5	/solr/clusterprops.json
   [junit4]   2> 	5	/solr/clusterstate.json
   [junit4]   2> 	2	/solr/overseer_elect/election/98386508336463878-127.0.0.1:61918_solr-n_0000000000
   [junit4]   2> 	2	/solr/collections/tlog_replica_test_create_delete/leader_elect/shard1/election/98386508336463877-core_node5-n_0000000001
   [junit4]   2> 	2	/solr/collections/tlog_replica_test_create_delete/leader_elect/shard1/election/98386508336463878-core_node9-n_0000000001
   [junit4]   2> 	2	/solr/collections/tlog_replica_test_create_delete/leader_elect/shard2/election/98386508336463878-core_node16-n_0000000000
   [junit4]   2> 	2	/solr/collections/tlog_replica_test_create_delete/leader_elect/shard2/election/98386508336463877-core_node16-n_0000000001
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	5	/solr/collections
   [junit4]   2> 	3	/solr/live_nodes
   [junit4]   2> 	2	/solr/overseer/queue
   [junit4]   2> 	2	/solr/overseer/collection-queue-work
   [junit4]   2> 
   [junit4]   2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestTlogReplica_7D00A8B605D57C02-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {foo=PostingsFormat(name=Memory), id=PostingsFormat(name=LuceneVarGapDocFreqInterval)}, docValues:{_version_=DocValuesFormat(name=Asserting), id=DocValuesFormat(name=Memory)}, maxPointsInLeafNode=1717, maxMBSortInHeap=5.807481830945461, sim=RandomSimilarity(queryNorm=true): {}, locale=fr-CA, timezone=Europe/Tallinn
   [junit4]   2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_141 (64-bit)/cpus=3,threads=1,free=159065184,total=518979584
   [junit4]   2> NOTE: All tests run in this JVM: [TestClusterStateMutator, SaslZkACLProviderTest, BigEndianAscendingWordDeserializerTest, TestLegacyField, TestMultiValuedNumericRangeQuery, TestInfoStreamLogging, CollectionsAPIAsyncDistributedZkTest, ChaosMonkeyNothingIsSafeWithPullReplicasTest, TestFuzzyAnalyzedSuggestions, TestSQLHandler, DistributedQueueTest, TestDistributedStatsComponentCardinality, DateFieldTest, TestStressLiveNodes, AnalyticsQueryTest, TestBulkSchemaConcurrent, TestCorePropertiesReload, TestSolrDeletionPolicy2, HttpPartitionTest, TestSchemaSimilarityResource, InfixSuggestersTest, TestHttpShardHandlerFactory, HdfsSyncSliceTest, BaseCdcrDistributedZkTest, TestReqParamsAPI, QueryParsingTest, TestSolrConfigHandler, TestDynamicFieldCollectionResource, MigrateRouteKeyTest, TimeZoneUtilsTest, TestCodecSupport, BooleanFieldTest, TestPullReplicaErrorHandling, RegexBoostProcessorTest, SolrShardReporterTest, TestImpersonationWithHadoopAuth, DocValuesNotIndexedTest, TestCloudPivotFacet, TestSurroundQueryParser, BlockJoinFacetRandomTest, SolrCloudExampleTest, HLLUtilTest, TestSha256AuthenticationProvider, FieldAnalysisRequestHandlerTest, CloudMLTQParserTest, TestSearcherReuse, TestStressReorder, RankQueryTest, TestCloudSchemaless, TestSchemalessBufferedUpdates, PeerSyncTest, TestComplexPhraseLeadingWildcard, ConcurrentDeleteAndCreateCollectionTest, TestConfigSetsAPIZkFailure, TestDFRSimilarityFactory, TestNumericTerms32, TestUpdate, AliasIntegrationTest, PrimitiveFieldTypeTest, TestXIncludeConfig, TestSolrCLIRunExample, SimpleFacetsTest, SuggesterWFSTTest, V2StandaloneTest, ZkCLITest, HardAutoCommitTest, TestBulkSchemaAPI, DistributedFacetPivotSmallTest, TestTrackingShardHandlerFactory, PrimUtilsTest, TestFoldingMultitermQuery, TestPHPSerializedResponseWriter, SolrCoreTest, BlobRepositoryCloudTest, TolerantUpdateProcessorTest, TestSmileRequest, TestIBSimilarityFactory, TestCustomDocTransformer, TestDistribIDF, MetricUtilsTest, TestUtils, TestHdfsBackupRestoreCore, TestExceedMaxTermLength, ResourceLoaderTest, ChaosMonkeyNothingIsSafeTest, DistributedFacetPivotLongTailTest, TestHashQParserPlugin, DistribDocExpirationUpdateProcessorTest, TestPointFields, TestDeleteCollectionOnDownNodes, TestConfigSetsAPI, RecoveryZkTest, TestNumericRangeQuery64, TestDownShardTolerantSearch, HdfsRecoveryZkTest, TestSubQueryTransformer, BinaryUpdateRequestHandlerTest, SolrTestCaseJ4Test, TestLegacyNumericUtils, BitVectorTest, TestStressVersions, CSVRequestHandlerTest, InfoHandlerTest, TestPseudoReturnFields, TestQuerySenderListener, ScriptEngineTest, PKIAuthenticationIntegrationTest, TestAtomicUpdateErrorCases, TestNoOpRegenerator, DistanceUnitsTest, TestJsonFacets, CircularListTest, MoveReplicaTest, TestRawResponseWriter, TestSweetSpotSimilarityFactory, TestHalfAndHalfDocValues, TestPayloadCheckQParserPlugin, TestRequestStatusCollectionAPI, TestReplicaProperties, CoreAdminHandlerTest, TestUninvertingReader, TestSolrCoreParser, TestJmxIntegration, HdfsChaosMonkeySafeLeaderTest, NoCacheHeaderTest, TestMissingGroups, ClusterStateUpdateTest, RAMDirectoryFactoryTest, TestManagedStopFilterFactory, ClusterStateTest, TestLocalFSCloudBackupRestore, DirectUpdateHandlerTest, TestQuerySenderNoQuery, SolrSlf4jReporterTest, TestStressCloudBlindAtomicUpdates, SolrIndexMetricsTest, HdfsNNFailoverTest, TestGeoJSONResponseWriter, UtilsToolTest, CachingDirectoryFactoryTest, DeleteLastCustomShardedReplicaTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, TestPhraseSuggestions, TestBinaryField, SpatialFilterTest, TestReplicationHandler, TestDynamicLoading, SortByFunctionTest, HdfsDirectoryTest, TestFaceting, HdfsWriteToMultipleCollectionsTest, DistributedSpellCheckComponentTest, TestLeaderElectionZkExpiry, TestPrepRecovery, DisMaxRequestHandlerTest, SharedFSAutoReplicaFailoverTest, IndexSchemaTest, BasicAuthIntegrationTest, TestJavabinTupleStreamParser, ClassificationUpdateProcessorIntegrationTest, TestRecoveryHdfs, TestSegmentSorting, MinimalSchemaTest, TestFieldCacheReopen, TestSort, HdfsTlogReplayBufferedWhileIndexingTest, DistanceFunctionTest, TestIndexingPerformance, DataDrivenBlockJoinTest, DistributedDebugComponentTest, TestWriterPerf, TestCollectionAPI, ZkStateWriterTest, TestFunctionQuery, AlternateDirectoryTest, DistributedQueryComponentCustomSortTest, ConjunctionSolrSpellCheckerTest, RequestHandlersTest, TestEmbeddedSolrServerSchemaAPI, ChaosMonkeySafeLeaderTest, BasicDistributedZk2Test, UnloadDistributedZkTest, SyncSliceTest, OverseerTest, LeaderElectionIntegrationTest, ShardRoutingTest, BasicZkTest, FullSolrCloudDistribCmdsTest, TestRandomFaceting, LeaderElectionTest, ZkSolrClientTest, TestHashPartitioner, ZkControllerTest, TestJoin, DistributedTermsComponentTest, QueryElevationComponentTest, TestFiltering, TestLazyCores, ShowFileRequestHandlerTest, SolrIndexSplitterTest, SimplePostToolTest, AnalysisAfterCoreReloadTest, TestExtendedDismaxParser, SolrRequestParserTest, DocValuesTest, SuggesterTSTTest, SuggesterTest, TestTrie, TestCSVLoader, PolyFieldTest, WordBreakSolrSpellCheckerTest, SchemaVersionSpecificBehaviorTest, SolrCoreCheckLockOnStartupTest, TestWordDelimiterFilterFactory, QueryEqualityTest, DirectUpdateHandlerOptimizeTest, DefaultValueUpdateProcessorTest, StatelessScriptUpdateProcessorFactoryTest, DocValuesMultiTest, SolrInfoBeanTest, DebugComponentTest, LukeRequestHandlerTest, TestOmitPositions, TermsComponentTest, PathHierarchyTokenizerFactoryTest, RequiredFieldsTest, IndexSchemaRuntimeFieldTest, TestCSVResponseWriter, UniqFieldsUpdateProcessorFactoryTest, TestPartialUpdateDeduplication, SearchHandlerTest, HighlighterConfigTest, SOLR749Test, UpdateParamsTest, TestSolrIndexConfig, CopyFieldTest, BadComponentTest, MultiTermTest, SampleTest, TestDocSet, TestSearchPerf, NumericFieldsTest, OutputWriterTest, ExternalFileFieldSortTest, TestSolrCoreProperties, TestPostingsSolrHighlighter, DirectSolrConnectionTest, NotRequiredUniqueKeyTest, TestLuceneMatchVersion, SpellPossibilityIteratorTest, TestCharFilters, SynonymTokenizerTest, TestPerFieldSimilarity, TestLMDirichletSimilarityFactory, TestBM25SimilarityFactory, TestFastWriter, TestFastOutputStream, URLClassifyProcessorTest, TestFastLRUCache, DateMathParserTest, PreAnalyzedFieldTest, TestSuggestSpellingConverter, DOMUtilTest, TestSolrJ, TestDocumentBuilder, ZkNodePropsTest, SliceStateTest, SystemInfoHandlerTest, UUIDFieldTest, TestRTGBase, CursorPagingTest, TestCrossCoreJoin, TestCursorMarkWithoutUniqueKey, TestDistributedMissingSort, TestHighlightDedupGrouping, TestSimpleTrackingShardHandler, TestJettySolrRunner, ConnectionReuseTest, AssignTest, CloudExitableDirectoryReaderTest, CollectionStateFormat2Test, CollectionTooManyReplicasTest, CreateCollectionCleanupTest, DeleteNodeTest, DeleteReplicaTest, DeleteShardTest, DeleteStatusTest, LeaderElectionContextKeyTest, LeaderFailoverAfterPartitionTest, LeaderInitiatedRecoveryOnShardRestartTest, MoveReplicaHDFSTest, MoveReplicaHDFSUlogDirTest, MultiThreadedOCPTest, ReplicationFactorTest, RestartWhileUpdatingTest, RollingRestartTest, SimpleCollectionCreateDeleteTest, SolrCLIZkUtilsTest, SolrXmlInZkTest, TestAuthenticationFramework, TestCloudInspectUtil, TestCloudJSONFacetJoinDomain, TestCloudPseudoReturnFields, TestCloudRecovery, TestClusterProperties, TestConfigSetsAPIExclusivity, TestHdfsCloudBackupRestore, TestLockTree, TestMiniSolrCloudClusterSSL, TestOnReconnectListenerSupport, TestRandomRequestDistribution, TestRebalanceLeaders, TestShortCircuitedRequests, TestSizeLimitedDistributedMap, TestSolrCloudWithKerberosAlt, TestTlogReplica]
   [junit4] Completed [726/726 (3!)] on J0 in 88.36s, 13 tests, 4 errors <<< FAILURES!

[...truncated 43320 lines...]

[JENKINS] Lucene-Solr-7.x-MacOSX (64bit/jdk1.8.0) - Build # 80 - Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-MacOSX/80/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseSerialGC

3 tests failed.
FAILED:  org.apache.solr.cloud.CustomCollectionTest.testRouteFieldForImplicitRouter

Error Message:
Collection not found: withShardField

Stack Trace:
org.apache.solr.common.SolrException: Collection not found: withShardField
	at __randomizedtesting.SeedInfo.seed([FAF597185B53831A:AFA57F8AF7AA4CEA]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.getCollectionNames(CloudSolrClient.java:1139)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:822)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
	at org.apache.solr.client.solrj.request.UpdateRequest.commit(UpdateRequest.java:233)
	at org.apache.solr.cloud.CustomCollectionTest.testRouteFieldForImplicitRouter(CustomCollectionTest.java:141)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.CustomCollectionTest.testCreateShardRepFactor

Error Message:
Error from server at http://127.0.0.1:55064/solr: Could not find collection : testCreateShardRepFactor

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:55064/solr: Could not find collection : testCreateShardRepFactor
	at __randomizedtesting.SeedInfo.seed([FAF597185B53831A:E1B1D9C430796DA]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:626)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:252)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:241)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1121)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:862)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
	at org.apache.solr.cloud.CustomCollectionTest.testCreateShardRepFactor(CustomCollectionTest.java:190)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.response.transform.TestSubQueryTransformerDistrib

Error Message:
Could not find collection:people

Stack Trace:
java.lang.AssertionError: Could not find collection:people
	at __randomizedtesting.SeedInfo.seed([FAF597185B53831A]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNotNull(Assert.java:526)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:155)
	at org.apache.solr.response.transform.TestSubQueryTransformerDistrib.setupCluster(TestSubQueryTransformerDistrib.java:87)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:847)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 11720 lines...]
   [junit4] Suite: org.apache.solr.cloud.CustomCollectionTest
   [junit4]   2> 1150596 INFO  (SUITE-CustomCollectionTest-seed#[FAF597185B53831A]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_FAF597185B53831A-001/init-core-data-001
   [junit4]   2> 1150596 WARN  (SUITE-CustomCollectionTest-seed#[FAF597185B53831A]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
   [junit4]   2> 1150596 INFO  (SUITE-CustomCollectionTest-seed#[FAF597185B53831A]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 1150599 INFO  (SUITE-CustomCollectionTest-seed#[FAF597185B53831A]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth
   [junit4]   2> 1150600 INFO  (SUITE-CustomCollectionTest-seed#[FAF597185B53831A]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 4 servers in /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_FAF597185B53831A-001/tempDir-001
   [junit4]   2> 1150600 INFO  (SUITE-CustomCollectionTest-seed#[FAF597185B53831A]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1150600 INFO  (Thread-1898) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1150600 INFO  (Thread-1898) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1150603 ERROR (Thread-1898) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 1150709 INFO  (SUITE-CustomCollectionTest-seed#[FAF597185B53831A]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:55062
   [junit4]   2> 1150735 INFO  (jetty-launcher-1748-thread-4) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 1150735 INFO  (jetty-launcher-1748-thread-2) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 1150735 INFO  (jetty-launcher-1748-thread-1) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 1150735 INFO  (jetty-launcher-1748-thread-3) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 1150737 INFO  (jetty-launcher-1748-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2795b83e{/solr,null,AVAILABLE}
   [junit4]   2> 1150737 INFO  (jetty-launcher-1748-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@3e5a77f1{HTTP/1.1,[http/1.1]}{127.0.0.1:55064}
   [junit4]   2> 1150737 INFO  (jetty-launcher-1748-thread-1) [    ] o.e.j.s.Server Started @1154130ms
   [junit4]   2> 1150737 INFO  (jetty-launcher-1748-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=55064}
   [junit4]   2> 1150737 ERROR (jetty-launcher-1748-thread-1) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1150738 INFO  (jetty-launcher-1748-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.1.0
   [junit4]   2> 1150738 INFO  (jetty-launcher-1748-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1150738 INFO  (jetty-launcher-1748-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 1150738 INFO  (jetty-launcher-1748-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-07-28T08:20:40.257Z
   [junit4]   2> 1150740 INFO  (jetty-launcher-1748-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@28e4976d{/solr,null,AVAILABLE}
   [junit4]   2> 1150740 INFO  (jetty-launcher-1748-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@1cae3d8e{HTTP/1.1,[http/1.1]}{127.0.0.1:55065}
   [junit4]   2> 1150740 INFO  (jetty-launcher-1748-thread-2) [    ] o.e.j.s.Server Started @1154134ms
   [junit4]   2> 1150741 INFO  (jetty-launcher-1748-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=55065}
   [junit4]   2> 1150741 ERROR (jetty-launcher-1748-thread-2) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1150741 INFO  (jetty-launcher-1748-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.1.0
   [junit4]   2> 1150741 INFO  (jetty-launcher-1748-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1150741 INFO  (jetty-launcher-1748-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 1150741 INFO  (jetty-launcher-1748-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-07-28T08:20:40.260Z
   [junit4]   2> 1150744 INFO  (jetty-launcher-1748-thread-4) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6436ef37{/solr,null,AVAILABLE}
   [junit4]   2> 1150745 INFO  (jetty-launcher-1748-thread-3) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@187e3632{/solr,null,AVAILABLE}
   [junit4]   2> 1150745 INFO  (jetty-launcher-1748-thread-4) [    ] o.e.j.s.AbstractConnector Started ServerConnector@347df66a{HTTP/1.1,[http/1.1]}{127.0.0.1:55067}
   [junit4]   2> 1150745 INFO  (jetty-launcher-1748-thread-4) [    ] o.e.j.s.Server Started @1154138ms
   [junit4]   2> 1150745 INFO  (jetty-launcher-1748-thread-4) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=55067}
   [junit4]   2> 1150745 INFO  (jetty-launcher-1748-thread-3) [    ] o.e.j.s.AbstractConnector Started ServerConnector@4c51c9ca{HTTP/1.1,[http/1.1]}{127.0.0.1:55069}
   [junit4]   2> 1150745 INFO  (jetty-launcher-1748-thread-3) [    ] o.e.j.s.Server Started @1154138ms
   [junit4]   2> 1150745 INFO  (jetty-launcher-1748-thread-3) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=55069}
   [junit4]   2> 1150745 ERROR (jetty-launcher-1748-thread-4) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1150745 ERROR (jetty-launcher-1748-thread-3) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1150746 INFO  (jetty-launcher-1748-thread-4) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.1.0
   [junit4]   2> 1150746 INFO  (jetty-launcher-1748-thread-4) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1150746 INFO  (jetty-launcher-1748-thread-4) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 1150746 INFO  (jetty-launcher-1748-thread-4) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-07-28T08:20:40.265Z
   [junit4]   2> 1150746 INFO  (jetty-launcher-1748-thread-3) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.1.0
   [junit4]   2> 1150746 INFO  (jetty-launcher-1748-thread-3) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1150746 INFO  (jetty-launcher-1748-thread-3) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 1150746 INFO  (jetty-launcher-1748-thread-3) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-07-28T08:20:40.265Z
   [junit4]   2> 1150750 INFO  (jetty-launcher-1748-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1150751 INFO  (jetty-launcher-1748-thread-4) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1150751 INFO  (jetty-launcher-1748-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1150752 INFO  (jetty-launcher-1748-thread-3) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1150767 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15d884884390002, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1150769 INFO  (jetty-launcher-1748-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:55062/solr
   [junit4]   2> 1150770 INFO  (jetty-launcher-1748-thread-4) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:55062/solr
   [junit4]   2> 1150771 INFO  (jetty-launcher-1748-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:55062/solr
   [junit4]   2> 1150771 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15d884884390004, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1150774 INFO  (jetty-launcher-1748-thread-3) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:55062/solr
   [junit4]   2> 1150777 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15d884884390005, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1150779 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15d884884390006, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1150780 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15d884884390007, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1150783 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15d884884390008, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1150941 INFO  (jetty-launcher-1748-thread-4) [n:127.0.0.1:55067_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1150941 INFO  (jetty-launcher-1748-thread-3) [n:127.0.0.1:55069_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1150942 INFO  (jetty-launcher-1748-thread-4) [n:127.0.0.1:55067_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:55067_solr
   [junit4]   2> 1150943 INFO  (jetty-launcher-1748-thread-3) [n:127.0.0.1:55069_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:55069_solr
   [junit4]   2> 1150943 INFO  (jetty-launcher-1748-thread-4) [n:127.0.0.1:55067_solr    ] o.a.s.c.Overseer Overseer (id=98384611909369866-127.0.0.1:55067_solr-n_0000000000) starting
   [junit4]   2> 1150945 INFO  (zkCallback-1770-thread-1-processing-n:127.0.0.1:55069_solr) [n:127.0.0.1:55069_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1150945 INFO  (zkCallback-1768-thread-1-processing-n:127.0.0.1:55067_solr) [n:127.0.0.1:55067_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1150958 INFO  (jetty-launcher-1748-thread-4) [n:127.0.0.1:55067_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:55067_solr
   [junit4]   2> 1150960 INFO  (zkCallback-1770-thread-1-processing-n:127.0.0.1:55069_solr) [n:127.0.0.1:55069_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1150960 INFO  (zkCallback-1768-thread-1-processing-n:127.0.0.1:55067_solr) [n:127.0.0.1:55067_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1151009 INFO  (jetty-launcher-1748-thread-1) [n:127.0.0.1:55064_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1151015 INFO  (jetty-launcher-1748-thread-1) [n:127.0.0.1:55064_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1151019 INFO  (jetty-launcher-1748-thread-1) [n:127.0.0.1:55064_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:55064_solr
   [junit4]   2> 1151021 INFO  (jetty-launcher-1748-thread-2) [n:127.0.0.1:55065_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1151027 INFO  (zkCallback-1770-thread-1-processing-n:127.0.0.1:55069_solr) [n:127.0.0.1:55069_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1151028 INFO  (zkCallback-1768-thread-1-processing-n:127.0.0.1:55067_solr) [n:127.0.0.1:55067_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1151028 INFO  (zkCallback-1767-thread-1-processing-n:127.0.0.1:55064_solr) [n:127.0.0.1:55064_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1151035 INFO  (zkCallback-1769-thread-1-processing-n:127.0.0.1:55065_solr) [n:127.0.0.1:55065_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1151036 INFO  (jetty-launcher-1748-thread-2) [n:127.0.0.1:55065_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1151038 INFO  (jetty-launcher-1748-thread-2) [n:127.0.0.1:55065_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:55065_solr
   [junit4]   2> 1151040 INFO  (zkCallback-1769-thread-1-processing-n:127.0.0.1:55065_solr) [n:127.0.0.1:55065_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1151040 INFO  (zkCallback-1770-thread-1-processing-n:127.0.0.1:55069_solr) [n:127.0.0.1:55069_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1151040 INFO  (zkCallback-1768-thread-1-processing-n:127.0.0.1:55067_solr) [n:127.0.0.1:55067_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1151040 INFO  (zkCallback-1767-thread-1-processing-n:127.0.0.1:55064_solr) [n:127.0.0.1:55064_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1151130 INFO  (jetty-launcher-1748-thread-4) [n:127.0.0.1:55067_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_55067.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@34bde4cd
   [junit4]   2> 1151130 INFO  (jetty-launcher-1748-thread-3) [n:127.0.0.1:55069_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_55069.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@34bde4cd
   [junit4]   2> 1151132 INFO  (jetty-launcher-1748-thread-2) [n:127.0.0.1:55065_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_55065.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@34bde4cd
   [junit4]   2> 1151132 INFO  (jetty-launcher-1748-thread-4) [n:127.0.0.1:55067_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_55067.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@34bde4cd
   [junit4]   2> 1151132 INFO  (jetty-launcher-1748-thread-4) [n:127.0.0.1:55067_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_55067.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@34bde4cd
   [junit4]   2> 1151133 INFO  (jetty-launcher-1748-thread-3) [n:127.0.0.1:55069_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_55069.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@34bde4cd
   [junit4]   2> 1151133 INFO  (jetty-launcher-1748-thread-3) [n:127.0.0.1:55069_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_55069.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@34bde4cd
   [junit4]   2> 1151133 INFO  (jetty-launcher-1748-thread-4) [n:127.0.0.1:55067_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_FAF597185B53831A-001/tempDir-001/node4/.
   [junit4]   2> 1151135 INFO  (jetty-launcher-1748-thread-2) [n:127.0.0.1:55065_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_55065.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@34bde4cd
   [junit4]   2> 1151135 INFO  (jetty-launcher-1748-thread-2) [n:127.0.0.1:55065_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_55065.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@34bde4cd
   [junit4]   2> 1151136 INFO  (jetty-launcher-1748-thread-3) [n:127.0.0.1:55069_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_FAF597185B53831A-001/tempDir-001/node3/.
   [junit4]   2> 1151137 INFO  (jetty-launcher-1748-thread-2) [n:127.0.0.1:55065_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_FAF597185B53831A-001/tempDir-001/node2/.
   [junit4]   2> 1151137 INFO  (jetty-launcher-1748-thread-1) [n:127.0.0.1:55064_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_55064.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@34bde4cd
   [junit4]   2> 1151139 INFO  (jetty-launcher-1748-thread-1) [n:127.0.0.1:55064_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_55064.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@34bde4cd
   [junit4]   2> 1151139 INFO  (jetty-launcher-1748-thread-1) [n:127.0.0.1:55064_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_55064.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@34bde4cd
   [junit4]   2> 1151140 INFO  (jetty-launcher-1748-thread-1) [n:127.0.0.1:55064_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_FAF597185B53831A-001/tempDir-001/node1/.
   [junit4]   2> 1151216 INFO  (SUITE-CustomCollectionTest-seed#[FAF597185B53831A]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 1151217 INFO  (SUITE-CustomCollectionTest-seed#[FAF597185B53831A]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:55062/solr ready
   [junit4]   2> 1151241 INFO  (TEST-CustomCollectionTest.testRouteFieldForHashRouter-seed#[FAF597185B53831A]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testRouteFieldForHashRouter
   [junit4]   2> 1151243 INFO  (TEST-CustomCollectionTest.testRouteFieldForHashRouter-seed#[FAF597185B53831A]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 1151246 INFO  (qtp309303977-9975) [n:127.0.0.1:55064_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=2&collection.configName=conf&maxShardsPerNode=3&router.field=shard_s&name=routeFieldColl&nrtReplicas=2&action=CREATE&numShards=4&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1151248 INFO  (OverseerThreadFactory-4150-thread-1-processing-n:127.0.0.1:55067_solr) [n:127.0.0.1:55067_solr    ] o.a.s.c.CreateCollectionCmd Create collection routeFieldColl
   [junit4]   2> 1151366 INFO  (OverseerStateUpdate-98384611909369866-127.0.0.1:55067_solr-n_0000000000) [n:127.0.0.1:55067_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"routeFieldColl",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"routeFieldColl_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:55064/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 1151370 INFO  (OverseerStateUpdate-98384611909369866-127.0.0.1:55067_solr-n_0000000000) [n:127.0.0.1:55067_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"routeFieldColl",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"routeFieldColl_shard1_replica_n2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:55067/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 1151373 INFO  (OverseerStateUpdate-98384611909369866-127.0.0.1:55067_solr-n_0000000000) [n:127.0.0.1:55067_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"routeFieldColl",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"routeFieldColl_shard2_replica_n4",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:55069/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 1151376 INFO  (OverseerStateUpdate-98384611909369866-127.0.0.1:55067_solr-n_0000000000) [n:127.0.0.1:55067_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"routeFieldColl",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"routeFieldColl_shard2_replica_n6",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:55065/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 1151378 INFO  (OverseerStateUpdate-98384611909369866-127.0.0.1:55067_solr-n_0000000000) [n:127.0.0.1:55067_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"routeFieldColl",
   [junit4]   2>   "shard":"shard3",
   [junit4]   2>   "core":"routeFieldColl_shard3_replica_n8",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:55064/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 1151381 INFO  (OverseerStateUpdate-98384611909369866-127.0.0.1:55067_solr-n_0000000000) [n:127.0.0.1:55067_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"routeFieldColl",
   [junit4]   2>   "shard":"shard3",
   [junit4]   2>   "core":"routeFieldColl_shard3_replica_n10",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:55067/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 1151384 INFO  (OverseerStateUpdate-98384611909369866-127.0.0.1:55067_solr-n_0000000000) [n:127.0.0.1:55067_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"routeFieldColl",
   [junit4]   2>   "shard":"shard4",
   [junit4]   2>   "core":"routeFieldColl_shard4_replica_n12",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:55069/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 1151387 INFO  (OverseerStateUpdate-98384611909369866-127.0.0.1:55067_solr-n_0000000000) [n:127.0.0.1:55067_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"routeFieldColl",
   [junit4]   2>   "shard":"shard4",
   [junit4]   2>   "core":"routeFieldColl_shard4_replica_n14",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:55065/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 1151593 INFO  (qtp309303977-9976) [n:127.0.0.1:55064_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=routeFieldColl_shard1_replica_n1&action=CREATE&numShards=4&collection=routeFieldColl&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1151595 INFO  (qtp309303977-9977) [n:127.0.0.1:55064_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node11&collection.configName=conf&newCollection=true&name=routeFieldColl_shard3_replica_n8&action=CREATE&numShards=4&collection=routeFieldColl&shard=shard3&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1151596 INFO  (qtp309303977-9976) [n:127.0.0.1:55064_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1151596 INFO  (qtp50413182-9973) [n:127.0.0.1:55067_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node5&collection.configName=conf&newCollection=true&name=routeFieldColl_shard1_replica_n2&action=CREATE&numShards=4&collection=routeFieldColl&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1151597 INFO  (qtp50413182-9991) [n:127.0.0.1:55067_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node13&collection.configName=conf&newCollection=true&name=routeFieldColl_shard3_replica_n10&action=CREATE&numShards=4&collection=routeFieldColl&shard=shard3&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1151597 INFO  (qtp754679732-10000) [n:127.0.0.1:55069_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node15&collection.configName=conf&newCollection=true&name=routeFieldColl_shard4_replica_n12&action=CREATE&numShards=4&collection=routeFieldColl&shard=shard4&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1151597 INFO  (qtp50413182-9973) [n:127.0.0.1:55067_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1151597 INFO  (qtp754679732-10000) [n:127.0.0.1:55069_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1151597 INFO  (qtp754679732-10001) [n:127.0.0.1:55069_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node7&collection.configName=conf&newCollection=true&name=routeFieldColl_shard2_replica_n4&action=CREATE&numShards=4&collection=routeFieldColl&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1151597 INFO  (qtp520485304-9984) [n:127.0.0.1:55065_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node9&collection.configName=conf&newCollection=true&name=routeFieldColl_shard2_replica_n6&action=CREATE&numShards=4&collection=routeFieldColl&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1151598 INFO  (qtp520485304-9984) [n:127.0.0.1:55065_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1151598 INFO  (qtp520485304-9985) [n:127.0.0.1:55065_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node16&collection.configName=conf&newCollection=true&name=routeFieldColl_shard4_replica_n14&action=CREATE&numShards=4&collection=routeFieldColl&shard=shard4&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1151730 INFO  (zkCallback-1768-thread-1-processing-n:127.0.0.1:55067_solr) [n:127.0.0.1:55067_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/routeFieldColl/state.json] for collection [routeFieldColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1151730 INFO  (zkCallback-1767-thread-1-processing-n:127.0.0.1:55064_solr) [n:127.0.0.1:55064_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/routeFieldColl/state.json] for collection [routeFieldColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1151731 INFO  (zkCallback-1770-thread-1-processing-n:127.0.0.1:55069_solr) [n:127.0.0.1:55069_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/routeFieldColl/state.json] for collection [routeFieldColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1151731 INFO  (zkCallback-1768-thread-2-processing-n:127.0.0.1:55067_solr) [n:127.0.0.1:55067_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/routeFieldColl/state.json] for collection [routeFieldColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1151731 INFO  (zkCallback-1769-thread-1-processing-n:127.0.0.1:55065_solr) [n:127.0.0.1:55065_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/routeFieldColl/state.json] for collection [routeFieldColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1151732 INFO  (zkCallback-1769-thread-2-processing-n:127.0.0.1:55065_solr) [n:127.0.0.1:55065_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/routeFieldColl/state.json] for collection [routeFieldColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1151733 INFO  (zkCallback-1767-thread-2-processing-n:127.0.0.1:55064_solr) [n:127.0.0.1:55064_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/routeFieldColl/state.json] for collection [routeFieldColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1152631 INFO  (qtp754679732-10001) [n:127.0.0.1:55069_solr c:routeFieldColl s:shard2 r:core_node7 x:routeFieldColl_shard2_replica_n4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 1152655 INFO  (qtp50413182-9973) [n:127.0.0.1:55067_solr c:routeFieldColl s:shard1 r:core_node5 x:routeFieldColl_shard1_replica_n2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 1152656 INFO  (qtp309303977-9977) [n:127.0.0.1:55064_solr c:routeFieldColl s:shard3 r:core_node11 x:routeFieldColl_shard3_replica_n8] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 1152659 INFO  (qtp309303977-9976) [n:127.0.0.1:55064_solr c:routeFieldColl s:shard1 r:core_node3 x:routeFieldColl_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 1152678 INFO  (qtp50413182-9991) [n:127.0.0.1:55067_solr c:routeFieldColl s:shard3 r:core_node13 x:routeFieldColl_shard3_replica_n10] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 1152680 INFO  (qtp520485304-9984) [n:127.0.0.1:55065_solr c:routeFieldColl s:shard2 r:core_node9 x:routeFieldColl_shard2_replica_n6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 1152684 INFO  (qtp520485304-9985) [n:127.0.0.1:55065_solr c:routeFieldColl s:shard4 r:core_node16 x:routeFieldColl_shard4_replica_n14] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 1152695 INFO  (qtp754679732-10000) [n:127.0.0.1:55069_solr c:routeFieldColl s:shard4 r:core_node15 x:routeFieldColl_shard4_replica_n12] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 1152698 INFO  (qtp754679732-10001) [n:127.0.0.1:55069_solr c:routeFieldColl s:shard2 r:core_node7 x:routeFieldColl_shard2_replica_n4] o.a.s.s.IndexSchema [routeFieldColl_shard2_replica_n4] Schema name=cloud-dynamic
   [junit4]   2> 1152701 INFO  (qtp309303977-9976) [n:127.0.0.1:55064_solr c:routeFieldColl s:shard1 r:core_node3 x:routeFieldColl_shard1_replica_n1] o.a.s.s.IndexSchema [routeFieldColl_shard1_replica_n1] Schema name=cloud-dynamic
   [junit4]   2> 1152705 INFO  (qtp50413182-9973) [n:127.0.0.1:55067_solr c:routeFieldColl s:shard1 r:core_node5 x:routeFieldColl_shard1_replica_n2] o.a.s.s.IndexSchema [routeFieldColl_shard1_replica_n2] Schema name=cloud-dynamic
   [junit4]   2> 1152706 INFO  (qtp309303977-9977) [n:127.0.0.1:55064_solr c:routeFieldColl s:shard3 r:core_node11 x:routeFieldColl_shard3_replica_n8] o.a.s.s.IndexSchema [routeFieldColl_shard3_replica_n8] Schema name=cloud-dynamic
   [junit4]   2> 1152720 INFO  (qtp520485304-9984) [n:127.0.0.1:55065_solr c:routeFieldColl s:shard2 r:core_node9 x:routeFieldColl_shard2_replica_n6] o.a.s.s.IndexSchema [routeFieldColl_shard2_replica_n6] Schema name=cloud-dynamic
   [junit4]   2> 1152742 INFO  (qtp50413182-9973) [n:127.0.0.1:55067_solr c:routeFieldColl s:shard1 r:core_node5 x:routeFieldColl_shard1_replica_n2] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
   [junit4]   2> 1152742 INFO  (qtp50413182-9973) [n:127.0.0.1:55067_solr c:routeFieldColl s:shard1 r:core_node5 x:routeFieldColl_shard1_replica_n2] o.a.s.c.CoreContainer Creating SolrCore 'routeFieldColl_shard1_replica_n2' using configuration from collection routeFieldColl, trusted=true
   [junit4]   2> 1152743 INFO  (qtp50413182-9973) [n:127.0.0.1:55067_solr c:routeFieldColl s:shard1 r:core_node5 x:routeFieldColl_shard1_replica_n2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_55067.solr.core.routeFieldColl.shard1.replica_n2' (registry 'solr.core.routeFieldColl.shard1.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@34bde4cd
   [junit4]   2> 1152743 INFO  (qtp50413182-9973) [n:127.0.0.1:55067_solr c:routeFieldColl s:shard1 r:core_node5 x:routeFieldColl_shard1_replica_n2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1152743 INFO  (qtp50413182-9973) [n:127.0.0.1:55067_solr c:routeFieldColl s:shard1 r:core_node5 x:routeFieldColl_shard1_replica_n2] o.a.s.c.SolrCore [[routeFieldColl_shard1_replica_n2] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_FAF597185B53831A-001/tempDir-001/node4/routeFieldColl_shard1_replica_n2], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_FAF597185B53831A-001/tempDir-001/node4/./routeFieldColl_shard1_replica_n2/data/]
   [junit4]   2> 1152743 INFO  (qtp309303977-9976) [n:127.0.0.1:55064_solr c:routeFieldColl s:shard1 r:core_node3 x:routeFieldColl_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
   [junit4]   2> 1152744 INFO  (qtp309303977-9976) [n:127.0.0.1:55064_solr c:routeFieldColl s:shard1 r:core_node3 x:routeFieldColl_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'routeFieldColl_shard1_replica_n1' using configuration from collection routeFieldColl, trusted=true
   [junit4]   2> 1152744 INFO  (qtp309303977-9976) [n:127.0.0.1:55064_solr c:routeFieldColl s:shard1 r:core_node3 x:routeFieldColl_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_55064.solr.core.routeFieldColl.shard1.replica_n1' (registry 'solr.core.routeFieldColl.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@34bde4cd
   [junit4]   2> 1152744 INFO  (qtp309303977-9976) [n:127.0.0.1:55064_solr c:routeFieldColl s:shard1 r:core_node3 x:routeFieldColl_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1152744 INFO  (qtp309303977-9976) [n:127.0.0.1:55064_solr c:routeFieldColl s:shard1 r:core_node3 x:routeFieldColl_shard1_replica_n1] o.a.s.c.SolrCore [[routeFieldColl_shard1_replica_n1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_FAF597185B53831A-001/tempDir-001/node1/routeFieldColl_shard1_replica_n1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_FAF597185B53831A-001/tempDir-001/node1/./routeFieldColl_shard1_replica_n1/data/]
   [junit4]   2> 1152756 INFO  (qtp520485304-9984) [n:127.0.0.1:55065_solr c:routeFieldColl s:shard2 r:core_node9 x:routeFieldColl_shard2_replica_n6] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
   [junit4]   2> 1152756 INFO  (qtp520485304-9984) [n:127.0.0.1:55065_solr c:routeFieldColl s:shard2 r:core_node9 x:routeFieldColl_shard2_replica_n6] o.a.s.c.CoreContainer Creating SolrCore 'routeFieldColl_shard2_replica_n6' using configuration from collection routeFieldColl, trusted=true
   [junit4]   2> 1152757 INFO  (qtp520485304-9984) [n:127.0.0.1:55065_solr c:routeFieldColl s:shard2 r:core_node9 x:routeFieldColl_shard2_replica_n6] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_55065.solr.core.routeFieldColl.shard2.replica_n6' (registry 'solr.core.routeFieldColl.shard2.replica_n6') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@34bde4cd
   [junit4]   2> 1152757 INFO  (qtp520485304-9984) [n:127.0.0.1:55065_solr c:routeFieldColl s:shard2 r:core_node9 x:routeFieldColl_shard2_replica_n6] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1152757 INFO  (qtp520485304-9984) [n:127.0.0.1:55065_solr c:routeFieldColl s:shard2 r:core_node9 x:routeFieldColl_shard2_replica_n6] o.a.s.c.SolrCore [[routeFieldColl_shard2_replica_n6] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_FAF597185B53831A-001/tempDir-001/node2/routeFieldColl_shard2_replica_n6], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_FAF597185B53831A-001/tempDir-001/node2/./routeFieldColl_shard2_replica_n6/data/]
   [junit4]   2> 1152758 INFO  (qtp50413182-9991) [n:127.0.0.1:55067_solr c:routeFieldColl s:shard3 r:core_node13 x:routeFieldColl_shard3_replica_n10] o.a.s.s.IndexSchema [routeFieldColl_shard3_replica_n10] Schema name=cloud-dynamic
   [junit4]   2> 1152782 INFO  (qtp309303977-9977) [n:127.0.0.1:55064_solr c:routeFieldColl s:shard3 r:core_node11 x:routeFieldColl_shard3_replica_n8] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
   [junit4]   2> 1152782 INFO  (qtp309303977-9977) [n:127.0.0.1:55064_solr c:routeFieldColl s:shard3 r:core_node11 x:routeFieldColl_shard3_replica_n8] o.a.s.c.CoreContainer Creating SolrCore 'routeFieldColl_shard3_replica_n8' using configuration from collection routeFieldColl, trusted=true
   [junit4]   2> 1152783 INFO  (qtp309303977-9977) [n:127.0.0.1:55064_solr c:routeFieldColl s:shard3 r:core_node11 x:routeFieldColl_shard3_replica_n8] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_55064.solr.core.routeFieldColl.shard3.replica_n8' (registry 'solr.core.routeFieldColl.shard3.replica_n8') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@34bde4cd
   [junit4]   2> 1152783 INFO  (qtp309303977-9977) [n:127.0.0.1:55064_solr c:routeFieldColl s:shard3 r:core_node11 x:routeFieldColl_shard3_replica_n8] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1152783 INFO  (qtp309303977-9977) [n:127.0.0.1:55064_solr c:routeFieldColl s:shard3 r:core_node11 x:routeFieldColl_shard3_replica_n8] o.a.s.c.SolrCore [[routeFieldColl_shard3_replica_n8] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_FAF597185B53831A-001/tempDir-001/node1/routeFieldColl_shard3_replica_n8], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_FAF597185B53831A-001/tempDir-001/node1/./routeFieldColl_shard3_replica_n8/data/]
   [junit4]   2> 1152788 INFO  (qtp754679732-10001) [n:127.0.0.1:55069_solr c:routeFieldColl s:shard2 r:core_node7 x:routeFieldColl_shard2_replica_n4] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
   [junit4]   2> 1152788 INFO  (qtp754679732-10001) [n:127.0.0.1:55069_solr c:routeFieldColl s:shard2 r:core_node7 x:routeFieldColl_shard2_replica_n4] o.a.s.c.CoreContainer Creating SolrCore 'routeFieldColl_shard2_replica_n4' using configuration from collection routeFieldColl, trusted=true
   [junit4]   2> 1152788 INFO  (qtp754679732-10001) [n:127.0.0.1:55069_solr c:routeFieldColl s:shard2 r:core_node7 x:routeFieldColl_shard2_replica_n4] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_55069.solr.core.routeFieldColl.shard2.replica_n4' (registry 'solr.core.routeFieldColl.shard2.replica_n4') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@34bde4cd
   [junit4]   2> 1152788 INFO  (qtp754679732-10001) [n:127.0.0.1:55069_solr c:routeFieldColl s:shard2 r:core_node7 x:routeFieldColl_shard2_replica_n4] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1152788 INFO  (qtp754679732-10001) [n:127.0.0.1:55069_solr c:routeFieldColl s:shard2 r:core_node7 x:routeFieldColl_shard2_replica_n4] o.a.s.c.SolrCore [[routeFieldColl_shard2_replica_n4] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_FAF597185B53831A-001/tempDir-001/node3/routeFieldColl_shard2_replica_n4], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_FAF597185B53831A-001/tempDir-001/node3/./routeFieldColl_shard2_replica_n4/data/]
   [junit4]   2> 1152793 INFO  (qtp754679732-10000) [n:127.0.0.1:55069_solr c:routeFieldColl s:shard4 r:core_node15 x:routeFieldColl_shard4_replica_n12] o.a.s.s.IndexSchema [routeFieldColl_shard4_replica_n12] Schema name=cloud-dynamic
   [junit4]   2> 1152793 INFO  (qtp50413182-9991) [n:127.0.0.1:55067_solr c:routeFieldColl s:shard3 r:core_node13 x:routeFieldColl_shard3_replica_n10] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
   [junit4]   2> 1152793 INFO  (qtp50413182-9991) [n:127.0.0.1:55067_solr c:routeFieldColl s:shard3 r:core_node13 x:routeFieldColl_shard3_replica_n10] o.a.s.c.CoreContainer Creating SolrCore 'routeFieldColl_shard3_replica_n10' using configuration from collection routeFieldColl, trusted=true
   [junit4]   2> 1152794 INFO  (qtp50413182-9991) [n:127.0.0.1:55067_solr c:routeFieldColl s:shard3 r:core_node13 x:routeFieldColl_shard3_replica_n10] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_55067.solr.core.routeFieldColl.shard3.replica_n10' (registry 'solr.core.routeFieldColl.shard3.replica_n10') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@34bde4cd
   [junit4]   2> 1152794 INFO  (qtp50413182-9991) [n:127.0.0.1:55067_solr c:routeFieldColl s:shard3 r:core_node13 x:routeFieldColl_shard3_replica_n10] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1152794 INFO  (qtp50413182-9991) [n:127.0.0.1:55067_solr c:routeFieldColl s:shard3 r:core_node13 x:routeFieldColl_shard3_replica_n10] o.a.s.c.SolrCore [[routeFieldColl_shard3_replica_n10] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_FAF597185B53831A-001/tempDir-001/node4/routeFieldColl_shard3_replica_n10], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_FAF597185B53831A-001/tempDir-001/node4/./routeFieldColl_shard3_replica_n10/data/]
   [junit4]   2> 1152798 INFO  (qtp520485304-9985) [n:127.0.0.1:55065_solr c:routeFieldColl s:shard4 r:core_node16 x:routeFieldColl_shard4_replica_n14] o.a.s.s.IndexSchema [routeFieldColl_shard4_replica_n14] Schema name=cloud-dynamic
   [junit4]   2> 1152829 INFO  (qtp754679732-10000) [n:127.0.0.1:55069_solr c:routeFieldColl s:shard4 r:core_node15 x:routeFieldColl_shard4_replica_n12] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
   [junit4]   2> 1152829 INFO  (qtp754679732-10000) [n:127.0.0.1:55069_solr c:routeFieldColl s:shard4 r:core_node15 x:routeFieldColl_shard4_replica_n12] o.a.s.c.CoreContainer Creating SolrCore 'routeFieldColl_shard4_replica_n12' using configuration from collection routeFieldColl, trusted=true
   [junit4]   2> 1152830 INFO  (qtp754679732-10000) [n:127.0.0.1:55069_solr c:routeFieldColl s:shard4 r:core_node15 x:routeFieldColl_shard4_replica_n12] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_55069.solr.core.routeFieldColl.shard4.replica_n12' (registry 'solr.core.routeFieldColl.shard4.replica_n12') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@34bde4cd
   [junit4]   2> 1152830 INFO  (qtp754679732-10000) [n:127.0.0.1:55069_solr c:routeFieldColl s:shard4 r:core_node15 x:routeFieldColl_shard4_replica_n12] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1152830 INFO  (qtp754679732-10000) [n:127.0.0.1:55069_solr c:routeFieldColl s:shard4 r:core_node15 x:routeFieldColl_shard4_replica_n12] o.a.s.c.SolrCore [[routeFieldColl_shard4_replica_n12] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_FAF597185B53831A-001/tempDir-001/node3/routeFieldColl_shard4_replica_n12], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_FAF597185B53831A-001/tempDir-001/node3/./routeFieldColl_shard4_replica_n12/data/]
   [junit4]   2> 1152831 INFO  (qtp520485304-9985) [n:127.0.0.1:55065_solr c:routeFieldColl s:shard4 r:core_node16 x:routeFieldColl_shard4_replica_n14] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
   [junit4]   2> 1152831 INFO  (qtp520485304-9985) [n:127.0.0.1:55065_solr c:routeFieldColl s:shard4 r:core_node16 x:routeFieldColl_shard4_replica_n14] o.a.s.c.CoreContainer Creating SolrCore 'routeFieldColl_shard4_replica_n14' using configuration from collection routeFieldColl, trusted=true
   [junit4]   2> 1152831 INFO  (qtp520485304-9985) [n:127.0.0.1:55065_solr c:routeFieldColl s:shard4 r:core_node16 x:routeFieldColl_shard4_replica_n14] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_55065.solr.core.routeFieldColl.shard4.replica_n14' (registry 'solr.core.routeFieldColl.shard4.replica_n14') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@34bde4cd
   [junit4]   2> 1152832 INFO  (qtp520485304-9985) [n:127.0.0.1:55065_solr c:routeFieldColl s:shard4 r:core_node16 x:routeFieldColl_shard4_replica_n14] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1152832 INFO  (qtp520485304-9985) [n:127.0.0.1:55065_solr c:routeFieldColl s:shard4 r:core_node16 x:routeFieldColl_shard4_replica_n14] o.a.s.c.SolrCore [[routeFieldColl_shard4_replica_n14] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_FAF597185B53831A-001/tempDir-001/node2/routeFieldColl_shard4_replica_n14], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_FAF597185B53831A-001/tempDir-001/node2/./routeFieldColl_shard4_replica_n14/data/]
   [junit4]   2> 1152948 INFO  (qtp309303977-9977) [n:127.0.0.1:55064_solr c:routeFieldColl s:shard3 r:core_node11 x:routeFieldColl_shard3_replica_n8] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1152948 INFO  (qtp309303977-9977) [n:127.0.0.1:55064_solr c:routeFieldColl s:shard3 r:core_node11 x:routeFieldColl_shard3_replica_n8] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1152948 INFO  (qtp309303977-9976) [n:127.0.0.1:55064_solr c:routeFieldColl s:shard1 r:core_node3 x:routeFieldColl_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1152949 INFO  (qtp309303977-9976) [n:127.0.0.1:55064_solr c:routeFieldColl s:shard1 r:core_node3 x:routeFieldColl_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1152950 INFO  (qtp309303977-9976) [n:127.0.0.1:55064_solr c:routeFieldColl s:shard1 r:core_node3 x:routeFieldColl_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1152950 INFO  (qtp309303977-9976) [n:127.0.0.1:55064_solr c:routeFieldColl s:shard1 r:core_node3 x:routeFieldColl_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1152950 INFO  (qtp309303977-9977) [n:127.0.0.1:55064_solr c:routeFieldColl s:shard3 r:core_node11 x:routeFieldColl_shard3_replica_n8] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1152950 INFO  (qtp309303977-9977) [n:127.0.0.1:55064_solr c:routeFieldColl s:shard3 r:core_node11 x:routeFieldColl_shard3_replica_n8] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1152966 INFO  (qtp50413182-9973) [n:127.0.0.1:55067_solr c:routeFieldColl s:shard1 r:core_node5 x:routeFieldColl_shard1_replica_n2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1152966 INFO  (qtp50413182-9973) [n:127.0.0.1:55067_solr c:routeFieldColl s:shard1 r:core_node5 x:routeFieldColl_shard1_replica_n2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1152972 INFO  (qtp50413182-9973) [n:127.0.0.1:55067_solr c:routeFieldColl s:shard1 r:core_node5 x:routeFieldColl_shard1_replica_n2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1152972 INFO  (qtp50413182-9973) [n:127.0.0.1:55067_solr c:routeFieldColl s:shard1 r:core_node5 x:routeFieldColl_shard1_replica_n2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1152981 INFO  (qtp520485304-9985) [n:127.0.0.1:55065_solr c:routeFieldColl s:shard4 r:core_node16 x:routeFieldColl_shard4_replica_n14] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1152981 INFO  (qtp520485304-9985) [n:127.0.0.1:55065_solr c:routeFieldColl s:shard4 r:core_node16 x:routeFieldColl_shard4_replica_n14] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1152983 INFO  (qtp520485304-9985) [n:127.0.0.1:55065_solr c:routeFieldColl s:shard4 r:core_node16 x:routeFieldColl_shard4_replica_n14] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1152983 INFO  (qtp520485304-9985) [n:127.0.0.1:55065_solr c:routeFieldColl s:shard4 r:core_node16 x:routeFieldColl_shard4_replica_n14] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1152983 INFO  (qtp309303977-9976) [n:127.0.0.1:55064_solr c:routeFieldColl s:shard1 r:core_node3 x:routeFieldColl_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@7a7facdc[routeFieldColl_shard1_replica_n1] main]
   [junit4]   2> 1152983 INFO  (qtp309303977-9977) [n:127.0.0.1:55064_solr c:routeFieldColl s:shard3 r:core_node11 x:routeFieldColl_shard3_replica_n8] o.a.s.s.SolrIndexSearcher Opening [Searcher@586cc2f[routeFieldColl_shard3_replica_n8] main]
   [junit4]   2> 1152985 INFO  (qtp309303977-9976) [n:127.0.0.1:55064_solr c:routeFieldColl s:shard1 r:core_node3 x:routeFieldColl_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 1152985 INFO  (qtp309303977-9977) [n:127.0.0.1:55064_solr c:routeFieldColl s:shard3 r:core_node11 x:routeFieldColl_shard3_replica_n8] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 1152986 INFO  (qtp309303977-9976) [n:127.0.0.1:55064_solr c:routeFieldColl s:shard1 r:core_node3 x:routeFieldColl_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 1152986 INFO  (qtp309303977-9976) [n:127.0.0.1:55064_solr c:routeFieldColl s:shard1 r:core_node3 x:routeFieldColl_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1152986 INFO  (qtp309303977-9976) [n:127.0.0.1:55064_solr c:routeFieldColl s:shard1 r:core_node3 x:routeFieldColl_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1574153793049722880
   [junit4]   2> 1152988 INFO  (qtp309303977-9977) [n:127.0.0.1:55064_solr c:routeFieldColl s:shard3 r:core_node11 x:routeFieldColl_shard3_replica_n8] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 1152988 INFO  (qtp309303977-9977) [n:127.0.0.1:55064_solr c:routeFieldColl s:shard3 r:core_node11 x:routeFieldColl_shard3_replica_n8] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1152988 INFO  (qtp309303977-9977) [n:127.0.0.1:55064_solr c:routeFieldColl s:shard3 r:core_node11 x:routeFieldColl_shard3_replica_n8] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1574153793051820032
   [junit4]   2> 1152988 INFO  (searcherExecutor-4160-thread-1-processing-n:127.0.0.1:55064_solr x:routeFieldColl_shard1_replica_n1 s:shard1 c:routeFieldColl r:core_node3) [n:127.0.0.1:55064_solr c:routeFieldColl s:shard1 r:core_node3 x:routeFieldColl_shard1_replica_n1] o.a.s.c.SolrCore [routeFieldColl_shard1_replica_n1] Registered new searcher Searcher@7a7facdc[routeFieldColl_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1152989 INFO  (searcherExecutor-4162-thread-1-processing-n:127.0.0.1:55064_solr x:routeFieldColl_shard3_replica_n8 s:shard3 c:routeFieldColl r:core_node11) [n:127.0.0.1:55064_solr c:routeFieldColl s:shard3 r:core_node11 x:routeFieldColl_shard3_replica_n8] o.a.s.c.SolrCore [routeFieldColl_shard3_replica_n8] Registered new searcher Searcher@586cc2f[routeFieldColl_shard3_replica_n8] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1152994 INFO  (qtp309303977-9976) [n:127.0.0.1:55064_solr c:routeFieldColl s:shard1 r:core_node3 x:routeFieldColl_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard1: total=2 found=1 timeoutin=9999ms
   [junit4]   2> 1152995 INFO  (qtp309303977-9977) [n:127.0.0.1:55064_solr c:routeFieldColl s:shard3 r:core_node11 x:routeFieldColl_shard3_replica_n8] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard3: total=2 found=1 timeoutin=9999ms
   [junit4]   2> 1153816 INFO  (qtp520485304-9984) [n:127.0.0.1:55065_solr c:routeFieldColl s:shard2 r:core_node9 x:routeFieldColl_shard2_replica_n6] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1153816 INFO  (qtp520485304-9984) [n:127.0.0.1:55065_solr c:routeFieldColl s:shard2 r:core_node9 x:routeFieldColl_shard2_replica_n6] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1153817 INFO  (qtp520485304-9984) [n:127.0.0.1:55065_solr c:routeFieldColl s:shard2 r:core_node9 x:routeFieldColl_shard2_replica_n6] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1153817 INFO  (qtp520485304-9984) [n:127.0.0.1:55065_solr c:routeFieldColl s:shard2 r:core_node9 x:routeFieldColl_shard2_replica_n6] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1153819 INFO  (qtp50413182-9991) [n:127.0.0.1:55067_solr c:routeFieldColl s:shard3 r:core_node13 x:routeFieldColl_shard3_replica_n10] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1153819 INFO  (qtp50413182-9991) [n:127.0.0.1:55067_solr c:routeFieldColl s:shard3 r:core_node13 x:routeFieldColl_shard3_replica_n10] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1153820 INFO  (qtp754679732-10001) [n:127.0.0.1:55069_solr c:routeFieldColl s:shard2 r:core_node7 x:routeFieldColl_shard2_replica_n4] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1153820 INFO  (qtp754679732-10001) [n:127.0.0.1:55069_solr c:routeFieldColl s:shard2 r:core_node7 x:routeFieldColl_shard2_replica_n4] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1153820 INFO  (qtp50413182-9991) [n:127.0.0.1:55067_solr c:routeFieldColl s:shard3 r:core_node13 x:routeFieldColl_shard3_replica_n10] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1153820 INFO  (qtp50413182-9991) [n:127.0.0.1:55067_solr c:routeFieldColl s:shard3 r:core_node13 x:routeFieldColl_shard3_replica_n10] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1153821 INFO  (qtp754679732-10001) [n:127.0.0.1:55069_solr c:routeFieldColl s:shard2 r:core_node7 x:routeFieldColl_shard2_replica_n4] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1153821 INFO  (qtp754679732-10001) [n:127.0.0.1:55069_solr c:routeFieldColl s:shard2 r:core_node7 x:routeFieldColl_shard2_replica_n4] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1153822 INFO  (zkCallback-1768-thread-2-processing-n:127.0.0.1:55067_solr) [n:127.0.0.1:55067_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/routeFieldColl/state.json] for collection [routeFieldColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1153822 INFO  (zkCallback-1768-thread-1-processing-n:127.0.0.1:55067_solr) [n:127.0.0.1:55067_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/routeFieldColl/state.json] for collection [routeFieldColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1153822 INFO  (zkCallback-1767-thread-2-processing-n:127.0.0.1:55064_solr) [n:127.0.0.1:55064_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/routeFieldColl/state.json] for collection [routeFieldColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1153822 INFO  (zkCallback-1767-thread-1-processing-n:127.0.0.1:55064_solr) [n:127.0.0.1:55064_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/routeFieldColl/state.json] for collection [routeFieldColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1153823 INFO  (zkCallback-1770-thread-1-processing-n:127.0.0.1:55069_solr) [n:127.0.0.1:55069_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/routeFieldColl/state.json] for collection [routeFieldColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1153823 INFO  (zkCallback-1769-thread-2-processing-n:127.0.0.1:55065_solr) [n:127.0.0.1:55065_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/routeFieldColl/state.json] for collection [routeFieldColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1153823 INFO  (zkCallback-1769-thread-1-processing-n:127.0.0.1:55065_solr) [n:127.0.0.1:55065_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/routeFieldColl/state.json] for collection [routeFieldColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1153824 INFO  (qtp50413182-9973) [n:127.0.0.1:55067_solr c:routeFieldColl s:shard1 r:core_node5 x:routeFieldColl_shard1_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@1ac639f[routeFieldColl_shard1_replica_n2] main]
   [junit4]   2> 1153827 INFO  (qtp50413182-9973) [n:127.0.0.1:55067_solr c:routeFieldColl s:shard1 r:core_node5 x:routeFieldColl_shard1_replica_n2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 1153832 INFO  (qtp50413182-9973) [n:127.0.0.1:55067_solr c:routeFieldColl s:shard1 r:core_node5 x:routeFieldColl_shard1_replica_n2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 1153832 INFO  (qtp50413182-9973) [n:127.0.0.1:55067_solr c:routeFieldColl s:shard1 r:core_node5 x:routeFieldColl_shard1_replica_n2] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1153832 INFO  (qtp50413182-9973) [n:127.0.0.1:55067_solr c:routeFieldColl s:shard1 r:core_node5 x:routeFieldColl_shard1_replica_n2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1574153793936818176
   [junit4]   2> 1153833 INFO  (searcherExecutor-4159-thread-1-processing-n:127.0.0.1:55067_solr x:routeFieldColl_shard1_replica_n2 s:shard1 c:routeFieldColl r:core_node5) [n:127.0.0.1:55067_solr c:routeFieldColl s:shard1 r:core_node5 x:routeFieldColl_shard1_replica_n2] o.a.s.c.SolrCore [routeFieldColl_shard1_replica_n2] Registered new searcher Searcher@1ac639f[routeFieldColl_shard1_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1153833 INFO  (qtp520485304-9985) [n:127.0.0.1:55065_solr c:routeFieldColl s:shard4 r:core_node16 x:routeFieldColl_shard4_replica_n14] o.a.s.s.SolrIndexSearcher Opening [Searcher@e7043c4[routeFieldColl_shard4_replica_n14] main]
   [junit4]   2> 1153835 INFO  (qtp520485304-9985) [n:127.0.0.1:55065_solr c:routeFieldColl s:shard4 r:core_node16 x:routeFieldColl_shard4_replica_n14] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 1153836 INFO  (qtp520485304-9985) [n:127.0.0.1:55065_solr c:routeFieldColl s:shard4 r:core_node16 x:routeFieldColl_shard4_replica_n14] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 1153836 INFO  (qtp520485304-9985) [n:127.0.0.1:55065_solr c:routeFieldColl s:shard4 r:core_node16 x:routeFieldColl_shard4_replica_n14] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1153836 INFO  (qtp520485304-9985) [n:127.0.0.1:55065_solr c:routeFieldColl s:shard4 r:core_node16 x:routeFieldColl_shard4_replica_n14] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1574153793941012480
   [junit4]   2> 1153837 INFO  (searcherExecutor-4166-thread-1-processing-n:127.0.0.1:55065_solr x:routeFieldColl_shard4_replica_n14 s:shard4 c:routeFieldColl r:core_node16) [n:127.0.0.1:55065_solr c:routeFieldColl s:shard4 r:core_node16 x:routeFieldColl_shard4_replica_n14] o.a.s.c.SolrCore [routeFieldColl_shard4_replica_n14] Registered new searcher Searcher@e7043c4[routeFieldColl_shard4_replica_n14] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1153840 INFO  (qtp520485304-9984) [n:127.0.0.1:55065_solr c:routeFieldColl s:shard2 r:core_node9 x:routeFieldColl_shard2_replica_n6] o.a.s.s.SolrIndexSearcher Opening [Searcher@3aa9d8a3[routeFieldColl_shard2_replica_n6] main]
   [junit4]   2> 1153840 INFO  (qtp50413182-9991) [n:127.0.0.1:55067_solr c:routeFieldColl s:shard3 r:core_node13 x:routeFieldColl_shard3_replica_n10] o.a.s.s.SolrIndexSearcher Opening [Searcher@7428c547[routeFieldColl_shard3_replica_n10] main]
   [junit4]   2> 1153841 INFO  (qtp520485304-9984) [n:127.0.0.1:55065_solr c:routeFieldColl s:shard2 r:core_node9 x:routeFieldColl_shard2_replica_n6] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 1153841 INFO  (qtp50413182-9991) [n:127.0.0.1:55067_solr c:routeFieldColl s:shard3 r:core_node13 x:routeFieldColl_shard3_replica_n10] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 1153841 INFO  (qtp520485304-9984) [n:127.0.0.1:55065_solr c:routeFieldColl s:shard2 r:core_node9 x:routeFieldColl_shard2_replica_n6] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 1153842 INFO  (qtp50413182-9991) [n:127.0.0.1:55067_solr c:routeFieldColl s:shard3 r:core_node13 x:routeFieldColl_shard3_replica_n10] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 1153842 INFO  (qtp520485304-9984) [n:127.0.0.1:55065_solr c:routeFieldColl s:shard2 r:core_node9 x:routeFieldColl_shard2_replica_n6] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1153842 INFO  (qtp520485304-9984) [n:127.0.0.1:55065_solr c:routeFieldColl s:shard2 r:core_node9 x:routeFieldColl_shard2_replica_n6] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1574153793947303936
   [junit4]   2> 1153842 INFO  (qtp50413182-9991) [n:127.0.0.1:55067_solr c:routeFieldColl s:shard3 r:core_node13 x:routeFieldColl_shard3_replica_n10] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1153842 INFO  (qtp50413182-9991) [n:127.0.0.1:55067_solr c:routeFieldColl s:shard3 r:core_node13 x:routeFieldColl_shard3_replica_n10] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1574153793947303936
   [junit4]   2> 1153842 INFO  (searcherExecutor-4161-thread-1-processing-n:127.0.0.1:55065_solr x:routeFieldColl_shard2_replica_n6 s:shard2 c:routeFieldColl r:core_node9) [n:127.0.0.1:55065_solr c:routeFieldColl s:shard2 r:core_node9 x:routeFieldColl_shard2_replica_n6] o.a.s.c.SolrCore [routeFieldColl_shard2_replica_n6] Registered new searcher Searcher@3aa9d8a3[routeFieldColl_shard2_replica_n6] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1153843 INFO  (searcherExecutor-4164-thread-1-processing-n:127.0.0.1:55067_solr x:routeFieldColl_shard3_replica_n10 s:shard3 c:routeFieldColl r:core_node13) [n:127.0.0.1:55067_solr c:routeFieldColl s:shard3 r:core_node13 x:routeFieldColl_shard3_replica_n10] o.a.s.c.SolrCore [routeFieldColl_shard3_replica_n10] Registered new searcher Searcher@7428c547[routeFieldColl_shard3_replica_n10] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1153844 INFO  (qtp520485304-9985) [n:127.0.0.1:55065_solr c:routeFieldColl s:shard4 r:core_node16 x:routeFieldColl_shard4_replica_n14] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard4: total=2 found=1 timeoutin=9999ms
   [junit4]   2> 1153846 INFO  (qtp754679732-10001) [n:127.0.0.1:55069_solr c:routeFieldColl s:shard2 r:core_node7 x:routeFieldColl_shard2_replica_n4] o.a.s.s.SolrIndexSearcher Opening [Searcher@3993ecff[routeFieldColl_shard2_replica_n4] main]
   [junit4]   2> 1153847 INFO  (qtp754679732-10000) [n:127.0.0.1:55069_solr c:routeFieldColl s:shard4 r:core_node15 x:routeFieldColl_shard4_replica_n12] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1153847 INFO  (qtp754679732-10000) [n:127.0.0.1:55069_solr c:routeFieldColl s:shard4 r:core_node15 x:routeFieldColl_shard4_replica_n12] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1153847 INFO  (qtp754679732-10001) [n:127.0.0.1:55069_solr c:routeFieldColl s:shard2 r:core_node7 x:routeFieldColl_shard2_replica_n4] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 1153848 INFO  (qtp754679732-10001) [n:127.0.0.1:55069_solr c:routeFieldColl s:shard2 r:core_node7 x:routeFieldColl_shard2_replica_n4] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 1153848 INFO  (qtp754679732-10001) [n:127.0.0.1:55069_solr c:routeFieldColl s:shard2 r:core_node7 x:routeFieldColl_shard2_replica_n4] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1153848 INFO  (qtp754679732-10001) [n:127.0.0.1:55069_solr c:routeFieldColl s:shard2 r:core_node7 x:routeFieldColl_shard2_replica_n4] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1574153793953595392
   [junit4]   2> 1153848 INFO  (qtp520485304-9984) [n:127.0.0.1:55065_solr c:routeFieldColl s:shard2 r:core_node9 x:routeFieldColl_shard2_replica_n6] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard2: total=2 found=1 timeoutin=9999ms
   [junit4]   2> 1153849 INFO  (searcherExecutor-4163-thread-1-processing-n:127.0.0.1:55069_solr x:routeFieldColl_shard2_replica_n4 s:shard2 c:routeFieldColl r:core_node7) [n:127.0.0.1:55069_solr c:routeFieldColl s:shard2 r:core_node7 x:routeFieldColl_shard2_replica_n4] o.a.s.c.SolrCore [routeFieldColl_shard2_replica_n4] Registered new searcher Searcher@3993ecff[routeFieldColl_shard2_replica_n4] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1153851 INFO  (qtp754679732-10000) [n:127.0.0.1:55069_solr c:routeFieldColl s:shard4 r:core_node15 x:routeFieldColl_shard4_replica_n12] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1153851 INFO  (qtp754679732-10000) [n:127.0.0.1:55069_solr c:routeFieldColl s:shard4 r:core_node15 x:routeFieldColl_shard4_replica_n12] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1153852 INFO  (qtp754679732-10000) [n:127.0.0.1:55069_solr c:routeFieldColl s:shard4 r:core_node15 x:routeFieldColl_shard4_replica_n12] o.a.s.s.SolrIndexSearcher Opening [Searcher@46ec6818[routeFieldColl_shard4_replica_n12] main]
   [junit4]   2> 1153853 INFO  (qtp754679732-10000) [n:127.0.0.1:55069_solr c:routeFieldColl s:shard4 r:core_node15 x:routeFieldColl_shard4_replica_n12] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 1153854 INFO  (qtp754679732-10000) [n:127.0.0.1:55069_solr c:routeFieldColl s:shard4 r:core_node15 x:routeFieldColl_shard4_replica_n12] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 1153854 INFO  (qtp754679732-10000) [n:127.0.0.1:55069_solr c:routeFieldColl s:shard4 r:core_node15 x:routeFieldColl_shard4_replica_n12] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1153854 INFO  (qtp754679732-10000) [n:127.0.0.1:55069_solr c:routeFieldColl s:shard4 r:core_node15 x:routeFieldColl_shard4_replica_n12] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1574153793959886848
   [junit4]   2> 1153855 INFO  (searcherExecutor-4165-thread-1-processing-n:127.0.0.1:55069_solr x:routeFieldColl_shard4_replica_n12 s:shard4 c:routeFieldColl r:core_node15) [n:127.0.0.1:55069_solr c:routeFieldColl s:shard4 r:core_node15 x:routeFieldColl_shard4_replica_n12] o.a.s.c.SolrCore [routeFieldColl_shard4_replica_n12] Registered new searcher Searcher@46ec6818[routeFieldColl_shard4_replica_n12] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1153955 INFO  (zkCallback-1768-thread-1-processing-n:127.0.0.1:55067_solr) [n:127.0.0.1:55067_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/routeFieldColl/state.json] for collection [routeFieldColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1153955 INFO  (zkCallback-1767-thread-1-processing-n:127.0.0.1:55064_solr) [n:127.0.0.1:55064_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/routeFieldColl/state.json] for collection [routeFieldColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1153956 INFO  (zkCallback-1767-thread-2-processing-n:127.0.0.1:55064_solr) [n:127.0.0.1:55064_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/routeFieldColl/state.json] for collection [routeFieldColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1153956 INFO  (zkCallback-1770-thread-1-processing-n:127.0.0.1:55069_solr) [n:127.0.0.1:55069_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/routeFieldColl/state.json] for collection [routeFieldColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1153955 INFO  (zkCallback-1768-thread-2-processing-n:127.0.0.1:55067_solr) [n:127.0.0.1:55067_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/routeFieldColl/state.json] for collection [routeFieldColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1153956 INFO  (zkCallback-1769-thread-2-processing-n:127.0.0.1:55065_solr) [n:127.0.0.1:55065_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/routeFieldColl/state.json] for collection [routeFieldColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1153956 INFO  (zkCallback-1769-thread-1-processing-n:127.0.0.1:55065_solr) [n:127.0.0.1:55065_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/routeFieldColl/state.json] for collection [routeFieldColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1154325 INFO  (qtp309303977-9976) [n:127.0.0.1:55064_solr c:routeFieldColl s:shard1 r:core_node3 x:routeFieldColl_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1154326 INFO  (qtp309303977-9977) [n:127.0.0.1:55064_solr c:routeFieldColl s:shard3 r:core_node11 x:routeFieldColl_shard3_replica_n8] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1154326 INFO  (qtp309303977-9976) [n:127.0.0.1:55064_solr c:routeFieldColl s:shard1 r:core_node3 x:routeFieldColl_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1154326 INFO  (qtp309303977-9977) [n:127.0.0.1:55064_solr c:routeFieldColl s:shard3 r:core_node11 x:routeFieldColl_shard3_replica_n8] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]

[...truncated too long message...]

ion expired for /live_nodes]
   [junit4]   2> 1312037 INFO  (jetty-closer-1798-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@7c8c3f69{/solr,null,UNAVAILABLE}
   [junit4]   2> 1312038 INFO  (zkCallback-1816-thread-1-processing-n:127.0.0.1:55271_solr) [n:127.0.0.1:55271_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (3)
   [junit4]   2> 1312038 INFO  (zkCallback-1823-thread-1-processing-n:127.0.0.1:55272_solr) [n:127.0.0.1:55272_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (3)
   [junit4]   2> 1312038 INFO  (zkCallback-1814-thread-1-processing-n:127.0.0.1:55275_solr) [n:127.0.0.1:55275_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (3)
   [junit4]   2> 1312039 INFO  (jetty-closer-1798-thread-3) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 1312039 INFO  (jetty-closer-1798-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 1312039 INFO  (zkCallback-1816-thread-3-processing-n:127.0.0.1:55271_solr) [n:127.0.0.1:55271_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:55271_solr
   [junit4]   2> 1312039 INFO  (jetty-closer-1798-thread-4) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 1312041 INFO  (jetty-closer-1798-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 1312041 INFO  (jetty-closer-1798-thread-4) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 1312041 INFO  (jetty-closer-1798-thread-3) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 1312044 WARN  (zkCallback-1816-thread-3-processing-n:127.0.0.1:55271_solr) [n:127.0.0.1:55271_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 1312044 WARN  (zkCallback-1823-thread-3-processing-n:127.0.0.1:55272_solr) [n:127.0.0.1:55272_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 1312045 INFO  (jetty-closer-1798-thread-4) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@79884429{/solr,null,UNAVAILABLE}
   [junit4]   2> 1312045 INFO  (jetty-closer-1798-thread-2) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@4e906814{/solr,null,UNAVAILABLE}
   [junit4]   2> 1313544 WARN  (zkCallback-1814-thread-1-processing-n:127.0.0.1:55275_solr) [n:127.0.0.1:55275_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 1313545 WARN  (zkCallback-1824-thread-1-processing-n:127.0.0.1:55274_solr) [n:127.0.0.1:55274_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 1313545 INFO  (jetty-closer-1798-thread-3) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@12d225cc{/solr,null,UNAVAILABLE}
   [junit4]   2> 1313546 INFO  (jetty-closer-1798-thread-5) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@76ec2e1c{/solr,null,UNAVAILABLE}
   [junit4]   2> 1313549 ERROR (SUITE-TestSubQueryTransformerDistrib-seed#[FAF597185B53831A]-worker) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 1313550 INFO  (SUITE-TestSubQueryTransformerDistrib-seed#[FAF597185B53831A]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:55269 55269
   [junit4]   2> 1313568 INFO  (Thread-1948) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:55269 55269
   [junit4]   2> 1313738 WARN  (Thread-1948) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	6	/solr/aliases.json
   [junit4]   2> 	5	/solr/security.json
   [junit4]   2> 	2	/solr/collections/people/state.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	20	/solr/collections/people/state.json
   [junit4]   2> 	6	/solr/clusterstate.json
   [junit4]   2> 	6	/solr/clusterprops.json
   [junit4]   2> 	2	/solr/overseer_elect/election/98384617526984714-127.0.0.1:55271_solr-n_0000000001
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	6	/solr/live_nodes
   [junit4]   2> 	6	/solr/collections
   [junit4]   2> 
   [junit4]   2> 1313739 INFO  (SUITE-TestSubQueryTransformerDistrib-seed#[FAF597185B53831A]-worker) [    ] o.a.s.SolrTestCaseJ4 ------------------------------------------------------- Done waiting for tracked resources to be released
   [junit4]   2> Jul 28, 2017 8:23:23 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=FastCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST, chunkSize=2, maxDocsPerChunk=973, blockSize=1), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST, chunkSize=2, blockSize=1)), sim=RandomSimilarity(queryNorm=true): {}, locale=en-US, timezone=AET
   [junit4]   2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_141 (64-bit)/cpus=3,threads=1,free=43552960,total=492376064
   [junit4]   2> NOTE: All tests run in this JVM: [TestSolrCloudSnapshots, AnalysisAfterCoreReloadTest, TestPrepRecovery, SyncSliceTest, TestCloudRecovery, RankQueryTest, UtilsToolTest, VMParamsZkACLAndCredentialsProvidersTest, CloudExitableDirectoryReaderTest, PKIAuthenticationIntegrationTest, BasicAuthIntegrationTest, DistributedSpellCheckComponentTest, QueryParsingTest, TestInPlaceUpdatesDistrib, ReplicationFactorTest, LeaderElectionTest, TestQueryWrapperFilter, TestManagedSchemaAPI, TestDistribIDF, TestSegmentSorting, DocValuesTest, TestDFRSimilarityFactory, DateFieldTest, IndexBasedSpellCheckerTest, ManagedSchemaRoundRobinCloudTest, SliceStateTest, TestDistributedSearch, TestBinaryField, HdfsDirectoryTest, AliasIntegrationTest, PeerSyncTest, DirectUpdateHandlerTest, OverseerTest, TestTolerantSearch, TestCloudPivotFacet, SortByFunctionTest, DistributedFacetPivotLongTailTest, RollingRestartTest, SaslZkACLProviderTest, ZkStateWriterTest, TestSolrDeletionPolicy2, MoveReplicaTest, ShufflingReplicaListTransformerTest, DistributedFacetPivotSmallTest, TestSmileRequest, TestClusterStateMutator, TestMacros, AlternateDirectoryTest, TestSurroundQueryParser, TestSerializedLuceneMatchVersion, CollectionsAPIAsyncDistributedZkTest, TestSolrQueryResponse, ZkControllerTest, CloneFieldUpdateProcessorFactoryTest, HighlighterMaxOffsetTest, SolrGraphiteReporterTest, TestConfigSetsAPIZkFailure, TestDeleteCollectionOnDownNodes, TestFoldingMultitermQuery, PathHierarchyTokenizerFactoryTest, TimeZoneUtilsTest, TestFaceting, TestLazyCores, TestPolicyCloud, UnloadDistributedZkTest, SharedFSAutoReplicaFailoverTest, InfoHandlerTest, ChaosMonkeyNothingIsSafeTest, TestJmxIntegration, TestSort, TestTrieFacet, TestLMDirichletSimilarityFactory, LargeFieldTest, ExitableDirectoryReaderTest, TestGeoJSONResponseWriter, DateMathParserTest, TestBlobHandler, InfixSuggestersTest, HdfsChaosMonkeySafeLeaderTest, DataDrivenBlockJoinTest, SolrTestCaseJ4Test, BinaryUpdateRequestHandlerTest, SpatialFilterTest, TestSearcherReuse, SolrShardReporterTest, FieldAnalysisRequestHandlerTest, TestSolrCloudWithKerberosAlt, RecoveryZkTest, DocValuesNotIndexedTest, RegexBoostProcessorTest, TestPseudoReturnFields, TestCorePropertiesReload, TestApiFramework, SuggesterTest, OverseerModifyCollectionTest, ConcurrentDeleteAndCreateCollectionTest, TestIndexingPerformance, TestStressReorder, TestConfigOverlay, ZkCLITest, TestSizeLimitedDistributedMap, TestUninvertingReader, TestShardHandlerFactory, SolrMetricsIntegrationTest, TestRawResponseWriter, DistanceFunctionTest, PropertiesRequestHandlerTest, HdfsWriteToMultipleCollectionsTest, TestHashQParserPlugin, TestDynamicLoading, TestJsonFacets, ClusterStateUpdateTest, NotRequiredUniqueKeyTest, TestCloudSchemaless, TestNumericTerms32, AnalysisErrorHandlingTest, TestPKIAuthenticationPlugin, DirectSolrConnectionTest, GraphQueryTest, TestRandomDVFaceting, CSVRequestHandlerTest, HttpPartitionTest, TestPhraseSuggestions, CustomCollectionTest, TestCSVResponseWriter, RequiredFieldsTest, TestSha256AuthenticationProvider, WrapperMergePolicyFactoryTest, TolerantUpdateProcessorTest, TestNonDefinedSimilarityFactory, BitVectorTest, URLClassifyProcessorTest, TestSubQueryTransformerDistrib]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestSubQueryTransformerDistrib -Dtests.seed=FAF597185B53831A -Dtests.slow=true -Dtests.locale=en-US -Dtests.timezone=AET -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   0.00s J0 | TestSubQueryTransformerDistrib (suite) <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: Could not find collection:people
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([FAF597185B53831A]:0)
   [junit4]    > 	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:155)
   [junit4]    > 	at org.apache.solr.response.transform.TestSubQueryTransformerDistrib.setupCluster(TestSubQueryTransformerDistrib.java:87)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4] Completed [309/726 (2!)] on J0 in 78.23s, 0 tests, 1 failure <<< FAILURES!

[...truncated 44731 lines...]