You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2017/03/19 11:47:38 UTC

[JENKINS] Lucene-Solr-6.x-MacOSX (64bit/jdk1.8.0) - Build # 770 - Still Unstable!

Build: https://jenkins.thetaphi.de/job/Lucene-Solr-6.x-MacOSX/770/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseParallelGC

1 tests failed.
FAILED:  org.apache.solr.cloud.CustomCollectionTest.testCustomCollectionsAPI

Error Message:
Error from server at https://127.0.0.1:57887/solr: Failed to create shard

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:57887/solr: Failed to create shard
	at __randomizedtesting.SeedInfo.seed([98F8C321050165A8:F2194D4A389BD3D0]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:610)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:279)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:268)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:435)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:387)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1376)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1127)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:1057)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:160)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:177)
	at org.apache.solr.cloud.CustomCollectionTest.testCustomCollectionsAPI(CustomCollectionTest.java:103)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 11717 lines...]
   [junit4] Suite: org.apache.solr.cloud.CustomCollectionTest
   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_98F8C321050165A8-001/init-core-data-001
   [junit4]   2> 1252257 WARN  (SUITE-CustomCollectionTest-seed#[98F8C321050165A8]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
   [junit4]   2> 1252258 INFO  (SUITE-CustomCollectionTest-seed#[98F8C321050165A8]-worker) [    ] o.a.s.SolrTestCaseJ4 Using TrieFields
   [junit4]   2> 1252271 INFO  (SUITE-CustomCollectionTest-seed#[98F8C321050165A8]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth
   [junit4]   2> 1252283 INFO  (SUITE-CustomCollectionTest-seed#[98F8C321050165A8]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 4 servers in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_98F8C321050165A8-001/tempDir-001
   [junit4]   2> 1252284 INFO  (SUITE-CustomCollectionTest-seed#[98F8C321050165A8]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1252284 INFO  (Thread-1756) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1252284 INFO  (Thread-1756) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1252388 INFO  (SUITE-CustomCollectionTest-seed#[98F8C321050165A8]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:57884
   [junit4]   2> 1252436 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15ae620b4180000, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1252512 INFO  (jetty-launcher-1260-thread-3) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 1252512 INFO  (jetty-launcher-1260-thread-1) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 1252512 INFO  (jetty-launcher-1260-thread-2) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 1252527 INFO  (jetty-launcher-1260-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@17f985db{/solr,null,AVAILABLE}
   [junit4]   2> 1252528 INFO  (jetty-launcher-1260-thread-3) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@710f38a{/solr,null,AVAILABLE}
   [junit4]   2> 1252528 INFO  (jetty-launcher-1260-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@60c4c8f2{/solr,null,AVAILABLE}
   [junit4]   2> 1252528 INFO  (jetty-launcher-1260-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@7bae0b00{SSL,[ssl, http/1.1]}{127.0.0.1:57886}
   [junit4]   2> 1252528 INFO  (jetty-launcher-1260-thread-1) [    ] o.e.j.s.Server Started @1257645ms
   [junit4]   2> 1252528 INFO  (jetty-launcher-1260-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@f80a180{SSL,[ssl, http/1.1]}{127.0.0.1:57888}
   [junit4]   2> 1252529 INFO  (jetty-launcher-1260-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=57886}
   [junit4]   2> 1252529 INFO  (jetty-launcher-1260-thread-2) [    ] o.e.j.s.Server Started @1257645ms
   [junit4]   2> 1252529 INFO  (jetty-launcher-1260-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=57888}
   [junit4]   2> 1252529 ERROR (jetty-launcher-1260-thread-1) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1252529 ERROR (jetty-launcher-1260-thread-2) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1252529 INFO  (jetty-launcher-1260-thread-3) [    ] o.e.j.s.AbstractConnector Started ServerConnector@a1a2733{SSL,[ssl, http/1.1]}{127.0.0.1:57887}
   [junit4]   2> 1252529 INFO  (jetty-launcher-1260-thread-3) [    ] o.e.j.s.Server Started @1257646ms
   [junit4]   2> 1252529 INFO  (jetty-launcher-1260-thread-3) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=57887}
   [junit4]   2> 1252529 INFO  (jetty-launcher-1260-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 6.6.0
   [junit4]   2> 1252529 ERROR (jetty-launcher-1260-thread-3) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1252529 INFO  (jetty-launcher-1260-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1252530 INFO  (jetty-launcher-1260-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1252530 INFO  (jetty-launcher-1260-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-03-19T10:33:07.593Z
   [junit4]   2> 1252530 INFO  (jetty-launcher-1260-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 6.6.0
   [junit4]   2> 1252530 INFO  (jetty-launcher-1260-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1252530 INFO  (jetty-launcher-1260-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1252530 INFO  (jetty-launcher-1260-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-03-19T10:33:07.593Z
   [junit4]   2> 1252531 INFO  (jetty-launcher-1260-thread-3) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 6.6.0
   [junit4]   2> 1252532 INFO  (jetty-launcher-1260-thread-3) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1252532 INFO  (jetty-launcher-1260-thread-3) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1252532 INFO  (jetty-launcher-1260-thread-3) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-03-19T10:33:07.595Z
   [junit4]   2> 1252542 INFO  (jetty-launcher-1260-thread-4) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 1252545 INFO  (jetty-launcher-1260-thread-4) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@73915db5{/solr,null,AVAILABLE}
   [junit4]   2> 1252546 INFO  (jetty-launcher-1260-thread-4) [    ] o.e.j.s.AbstractConnector Started ServerConnector@2f83b55b{SSL,[ssl, http/1.1]}{127.0.0.1:57889}
   [junit4]   2> 1252547 INFO  (jetty-launcher-1260-thread-4) [    ] o.e.j.s.Server Started @1257663ms
   [junit4]   2> 1252632 INFO  (jetty-launcher-1260-thread-4) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=57889}
   [junit4]   2> 1252632 ERROR (jetty-launcher-1260-thread-4) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1252633 INFO  (jetty-launcher-1260-thread-4) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 6.6.0
   [junit4]   2> 1252633 INFO  (jetty-launcher-1260-thread-4) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1252633 INFO  (jetty-launcher-1260-thread-4) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1252633 INFO  (jetty-launcher-1260-thread-4) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-03-19T10:33:07.696Z
   [junit4]   2> 1252639 INFO  (jetty-launcher-1260-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1252639 INFO  (jetty-launcher-1260-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1252644 INFO  (jetty-launcher-1260-thread-3) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1252655 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15ae620b4180002, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1252661 INFO  (jetty-launcher-1260-thread-3) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1252661 INFO  (jetty-launcher-1260-thread-2) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1252661 INFO  (jetty-launcher-1260-thread-1) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1252662 INFO  (jetty-launcher-1260-thread-3) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:57884/solr
   [junit4]   2> 1252662 INFO  (jetty-launcher-1260-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:57884/solr
   [junit4]   2> 1252664 INFO  (jetty-launcher-1260-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:57884/solr
   [junit4]   2> 1252667 INFO  (jetty-launcher-1260-thread-4) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1252671 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15ae620b4180005, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1252679 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15ae620b4180004, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1252681 INFO  (jetty-launcher-1260-thread-4) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1252683 INFO  (jetty-launcher-1260-thread-4) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:57884/solr
   [junit4]   2> 1252752 INFO  (jetty-launcher-1260-thread-3) [n:127.0.0.1:57887_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1252754 INFO  (jetty-launcher-1260-thread-1) [n:127.0.0.1:57886_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1252754 INFO  (jetty-launcher-1260-thread-2) [n:127.0.0.1:57888_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1252755 INFO  (jetty-launcher-1260-thread-4) [n:127.0.0.1:57889_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1252756 INFO  (jetty-launcher-1260-thread-3) [n:127.0.0.1:57887_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:57887_solr
   [junit4]   2> 1252758 INFO  (jetty-launcher-1260-thread-2) [n:127.0.0.1:57888_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57888_solr
   [junit4]   2> 1252758 INFO  (jetty-launcher-1260-thread-1) [n:127.0.0.1:57886_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57886_solr
   [junit4]   2> 1252760 INFO  (jetty-launcher-1260-thread-4) [n:127.0.0.1:57889_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57889_solr
   [junit4]   2> 1252762 INFO  (jetty-launcher-1260-thread-3) [n:127.0.0.1:57887_solr    ] o.a.s.c.Overseer Overseer (id=97643370076700680-127.0.0.1:57887_solr-n_0000000000) starting
   [junit4]   2> 1252764 INFO  (zkCallback-1278-thread-1-processing-n:127.0.0.1:57888_solr) [n:127.0.0.1:57888_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1252765 INFO  (zkCallback-1282-thread-1-processing-n:127.0.0.1:57889_solr) [n:127.0.0.1:57889_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1252766 INFO  (zkCallback-1277-thread-1-processing-n:127.0.0.1:57886_solr) [n:127.0.0.1:57886_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1252767 INFO  (zkCallback-1276-thread-1-processing-n:127.0.0.1:57887_solr) [n:127.0.0.1:57887_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1252774 INFO  (zkCallback-1278-thread-1-processing-n:127.0.0.1:57888_solr) [n:127.0.0.1:57888_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1252775 INFO  (zkCallback-1282-thread-1-processing-n:127.0.0.1:57889_solr) [n:127.0.0.1:57889_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1252776 INFO  (zkCallback-1277-thread-1-processing-n:127.0.0.1:57886_solr) [n:127.0.0.1:57886_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1252777 INFO  (zkCallback-1276-thread-1-processing-n:127.0.0.1:57887_solr) [n:127.0.0.1:57887_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1252819 INFO  (jetty-launcher-1260-thread-3) [n:127.0.0.1:57887_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57887_solr
   [junit4]   2> 1252824 INFO  (zkCallback-1282-thread-1-processing-n:127.0.0.1:57889_solr) [n:127.0.0.1:57889_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1252825 INFO  (zkCallback-1277-thread-1-processing-n:127.0.0.1:57886_solr) [n:127.0.0.1:57886_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1252825 INFO  (zkCallback-1278-thread-1-processing-n:127.0.0.1:57888_solr) [n:127.0.0.1:57888_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1252825 INFO  (zkCallback-1276-thread-1-processing-n:127.0.0.1:57887_solr) [n:127.0.0.1:57887_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1253110 INFO  (jetty-launcher-1260-thread-4) [n:127.0.0.1:57889_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_98F8C321050165A8-001/tempDir-001/node4/.
   [junit4]   2> 1253114 INFO  (jetty-launcher-1260-thread-1) [n:127.0.0.1:57886_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_98F8C321050165A8-001/tempDir-001/node1/.
   [junit4]   2> 1253114 INFO  (jetty-launcher-1260-thread-2) [n:127.0.0.1:57888_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_98F8C321050165A8-001/tempDir-001/node2/.
   [junit4]   2> 1253118 INFO  (jetty-launcher-1260-thread-3) [n:127.0.0.1:57887_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_98F8C321050165A8-001/tempDir-001/node3/.
   [junit4]   2> 1253493 INFO  (SUITE-CustomCollectionTest-seed#[98F8C321050165A8]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 1253495 INFO  (SUITE-CustomCollectionTest-seed#[98F8C321050165A8]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:57884/solr ready
   [junit4]   2> 1253536 INFO  (TEST-CustomCollectionTest.testCreateShardRepFactor-seed#[98F8C321050165A8]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testCreateShardRepFactor
   [junit4]   2> 1253538 INFO  (TEST-CustomCollectionTest.testCreateShardRepFactor-seed#[98F8C321050165A8]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 1253587 INFO  (qtp456925188-7414) [n:127.0.0.1:57887_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params shards=a,b&replicationFactor=1&collection.configName=conf&name=testCreateShardRepFactor&router.name=implicit&action=CREATE&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1253591 INFO  (OverseerThreadFactory-3713-thread-1-processing-n:127.0.0.1:57887_solr) [n:127.0.0.1:57887_solr    ] o.a.s.c.CreateCollectionCmd Create collection testCreateShardRepFactor
   [junit4]   2> 1253736 INFO  (qtp844286241-7440) [n:127.0.0.1:57889_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=testCreateShardRepFactor_a_replica1&action=CREATE&numShards=2&collection=testCreateShardRepFactor&shard=a&wt=javabin&version=2
   [junit4]   2> 1253745 INFO  (qtp1255938767-7420) [n:127.0.0.1:57886_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=testCreateShardRepFactor_b_replica1&action=CREATE&numShards=2&collection=testCreateShardRepFactor&shard=b&wt=javabin&version=2
   [junit4]   2> 1253862 INFO  (zkCallback-1282-thread-1-processing-n:127.0.0.1:57889_solr) [n:127.0.0.1:57889_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCreateShardRepFactor/state.json] for collection [testCreateShardRepFactor] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1253862 INFO  (zkCallback-1277-thread-1-processing-n:127.0.0.1:57886_solr) [n:127.0.0.1:57886_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCreateShardRepFactor/state.json] for collection [testCreateShardRepFactor] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1254761 INFO  (qtp844286241-7440) [n:127.0.0.1:57889_solr c:testCreateShardRepFactor s:a  x:testCreateShardRepFactor_a_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.6.0
   [junit4]   2> 1254783 INFO  (qtp1255938767-7420) [n:127.0.0.1:57886_solr c:testCreateShardRepFactor s:b  x:testCreateShardRepFactor_b_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.6.0
   [junit4]   2> 1254791 INFO  (qtp844286241-7440) [n:127.0.0.1:57889_solr c:testCreateShardRepFactor s:a  x:testCreateShardRepFactor_a_replica1] o.a.s.s.IndexSchema [testCreateShardRepFactor_a_replica1] Schema name=cloud-dynamic
   [junit4]   2> 1254797 INFO  (qtp1255938767-7420) [n:127.0.0.1:57886_solr c:testCreateShardRepFactor s:b  x:testCreateShardRepFactor_b_replica1] o.a.s.s.IndexSchema [testCreateShardRepFactor_b_replica1] Schema name=cloud-dynamic
   [junit4]   2> 1254864 WARN  (qtp1255938767-7420) [n:127.0.0.1:57886_solr c:testCreateShardRepFactor s:b  x:testCreateShardRepFactor_b_replica1] o.a.s.s.IndexSchema [testCreateShardRepFactor_b_replica1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 1254865 INFO  (qtp1255938767-7420) [n:127.0.0.1:57886_solr c:testCreateShardRepFactor s:b  x:testCreateShardRepFactor_b_replica1] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
   [junit4]   2> 1254867 INFO  (qtp1255938767-7420) [n:127.0.0.1:57886_solr c:testCreateShardRepFactor s:b  x:testCreateShardRepFactor_b_replica1] o.a.s.c.CoreContainer Creating SolrCore 'testCreateShardRepFactor_b_replica1' using configuration from collection testCreateShardRepFactor
   [junit4]   2> 1254868 WARN  (qtp844286241-7440) [n:127.0.0.1:57889_solr c:testCreateShardRepFactor s:a  x:testCreateShardRepFactor_a_replica1] o.a.s.s.IndexSchema [testCreateShardRepFactor_a_replica1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 1254868 INFO  (qtp1255938767-7420) [n:127.0.0.1:57886_solr c:testCreateShardRepFactor s:b  x:testCreateShardRepFactor_b_replica1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1254868 INFO  (qtp1255938767-7420) [n:127.0.0.1:57886_solr c:testCreateShardRepFactor s:b  x:testCreateShardRepFactor_b_replica1] o.a.s.c.SolrCore [[testCreateShardRepFactor_b_replica1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_98F8C321050165A8-001/tempDir-001/node1/testCreateShardRepFactor_b_replica1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_98F8C321050165A8-001/tempDir-001/node1/./testCreateShardRepFactor_b_replica1/data/]
   [junit4]   2> 1254869 INFO  (qtp844286241-7440) [n:127.0.0.1:57889_solr c:testCreateShardRepFactor s:a  x:testCreateShardRepFactor_a_replica1] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
   [junit4]   2> 1254870 INFO  (qtp844286241-7440) [n:127.0.0.1:57889_solr c:testCreateShardRepFactor s:a  x:testCreateShardRepFactor_a_replica1] o.a.s.c.CoreContainer Creating SolrCore 'testCreateShardRepFactor_a_replica1' using configuration from collection testCreateShardRepFactor
   [junit4]   2> 1254871 INFO  (qtp844286241-7440) [n:127.0.0.1:57889_solr c:testCreateShardRepFactor s:a  x:testCreateShardRepFactor_a_replica1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1254871 INFO  (qtp844286241-7440) [n:127.0.0.1:57889_solr c:testCreateShardRepFactor s:a  x:testCreateShardRepFactor_a_replica1] o.a.s.c.SolrCore [[testCreateShardRepFactor_a_replica1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_98F8C321050165A8-001/tempDir-001/node4/testCreateShardRepFactor_a_replica1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_98F8C321050165A8-001/tempDir-001/node4/./testCreateShardRepFactor_a_replica1/data/]
   [junit4]   2> 1255032 INFO  (qtp844286241-7440) [n:127.0.0.1:57889_solr c:testCreateShardRepFactor s:a  x:testCreateShardRepFactor_a_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1255033 INFO  (qtp844286241-7440) [n:127.0.0.1:57889_solr c:testCreateShardRepFactor s:a  x:testCreateShardRepFactor_a_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1255034 INFO  (qtp1255938767-7420) [n:127.0.0.1:57886_solr c:testCreateShardRepFactor s:b  x:testCreateShardRepFactor_b_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1255034 INFO  (qtp1255938767-7420) [n:127.0.0.1:57886_solr c:testCreateShardRepFactor s:b  x:testCreateShardRepFactor_b_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1255035 INFO  (qtp844286241-7440) [n:127.0.0.1:57889_solr c:testCreateShardRepFactor s:a  x:testCreateShardRepFactor_a_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1255035 INFO  (qtp844286241-7440) [n:127.0.0.1:57889_solr c:testCreateShardRepFactor s:a  x:testCreateShardRepFactor_a_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1255036 INFO  (qtp1255938767-7420) [n:127.0.0.1:57886_solr c:testCreateShardRepFactor s:b  x:testCreateShardRepFactor_b_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1255036 INFO  (qtp1255938767-7420) [n:127.0.0.1:57886_solr c:testCreateShardRepFactor s:b  x:testCreateShardRepFactor_b_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1255068 INFO  (qtp844286241-7440) [n:127.0.0.1:57889_solr c:testCreateShardRepFactor s:a  x:testCreateShardRepFactor_a_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3d57b6e5[testCreateShardRepFactor_a_replica1] main]
   [junit4]   2> 1255069 INFO  (qtp1255938767-7420) [n:127.0.0.1:57886_solr c:testCreateShardRepFactor s:b  x:testCreateShardRepFactor_b_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@c793113[testCreateShardRepFactor_b_replica1] main]
   [junit4]   2> 1255071 INFO  (qtp844286241-7440) [n:127.0.0.1:57889_solr c:testCreateShardRepFactor s:a  x:testCreateShardRepFactor_a_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 1255071 INFO  (qtp1255938767-7420) [n:127.0.0.1:57886_solr c:testCreateShardRepFactor s:b  x:testCreateShardRepFactor_b_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 1255071 INFO  (qtp844286241-7440) [n:127.0.0.1:57889_solr c:testCreateShardRepFactor s:a  x:testCreateShardRepFactor_a_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 1255072 INFO  (qtp844286241-7440) [n:127.0.0.1:57889_solr c:testCreateShardRepFactor s:a  x:testCreateShardRepFactor_a_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1255073 INFO  (searcherExecutor-3723-thread-1-processing-n:127.0.0.1:57889_solr x:testCreateShardRepFactor_a_replica1 s:a c:testCreateShardRepFactor) [n:127.0.0.1:57889_solr c:testCreateShardRepFactor s:a  x:testCreateShardRepFactor_a_replica1] o.a.s.c.SolrCore [testCreateShardRepFactor_a_replica1] Registered new searcher Searcher@3d57b6e5[testCreateShardRepFactor_a_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1255073 INFO  (qtp1255938767-7420) [n:127.0.0.1:57886_solr c:testCreateShardRepFactor s:b  x:testCreateShardRepFactor_b_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 1255073 INFO  (qtp844286241-7440) [n:127.0.0.1:57889_solr c:testCreateShardRepFactor s:a  x:testCreateShardRepFactor_a_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1562293924146446336
   [junit4]   2> 1255073 INFO  (qtp1255938767-7420) [n:127.0.0.1:57886_solr c:testCreateShardRepFactor s:b  x:testCreateShardRepFactor_b_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1255073 INFO  (searcherExecutor-3722-thread-1-processing-n:127.0.0.1:57886_solr x:testCreateShardRepFactor_b_replica1 s:b c:testCreateShardRepFactor) [n:127.0.0.1:57886_solr c:testCreateShardRepFactor s:b  x:testCreateShardRepFactor_b_replica1] o.a.s.c.SolrCore [testCreateShardRepFactor_b_replica1] Registered new searcher Searcher@c793113[testCreateShardRepFactor_b_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1255073 INFO  (qtp1255938767-7420) [n:127.0.0.1:57886_solr c:testCreateShardRepFactor s:b  x:testCreateShardRepFactor_b_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1562293924146446336
   [junit4]   2> 1255088 INFO  (qtp844286241-7440) [n:127.0.0.1:57889_solr c:testCreateShardRepFactor s:a  x:testCreateShardRepFactor_a_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1255088 INFO  (qtp844286241-7440) [n:127.0.0.1:57889_solr c:testCreateShardRepFactor s:a  x:testCreateShardRepFactor_a_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1255088 INFO  (qtp844286241-7440) [n:127.0.0.1:57889_solr c:testCreateShardRepFactor s:a  x:testCreateShardRepFactor_a_replica1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:57889/solr/testCreateShardRepFactor_a_replica1/
   [junit4]   2> 1255088 INFO  (qtp844286241-7440) [n:127.0.0.1:57889_solr c:testCreateShardRepFactor s:a  x:testCreateShardRepFactor_a_replica1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 1255088 INFO  (qtp844286241-7440) [n:127.0.0.1:57889_solr c:testCreateShardRepFactor s:a  x:testCreateShardRepFactor_a_replica1] o.a.s.c.SyncStrategy https://127.0.0.1:57889/solr/testCreateShardRepFactor_a_replica1/ has no replicas
   [junit4]   2> 1255088 INFO  (qtp844286241-7440) [n:127.0.0.1:57889_solr c:testCreateShardRepFactor s:a  x:testCreateShardRepFactor_a_replica1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 1255094 INFO  (qtp1255938767-7420) [n:127.0.0.1:57886_solr c:testCreateShardRepFactor s:b  x:testCreateShardRepFactor_b_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1255094 INFO  (qtp1255938767-7420) [n:127.0.0.1:57886_solr c:testCreateShardRepFactor s:b  x:testCreateShardRepFactor_b_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1255094 INFO  (qtp1255938767-7420) [n:127.0.0.1:57886_solr c:testCreateShardRepFactor s:b  x:testCreateShardRepFactor_b_replica1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:57886/solr/testCreateShardRepFactor_b_replica1/
   [junit4]   2> 1255095 INFO  (qtp1255938767-7420) [n:127.0.0.1:57886_solr c:testCreateShardRepFactor s:b  x:testCreateShardRepFactor_b_replica1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 1255095 INFO  (qtp1255938767-7420) [n:127.0.0.1:57886_solr c:testCreateShardRepFactor s:b  x:testCreateShardRepFactor_b_replica1] o.a.s.c.SyncStrategy https://127.0.0.1:57886/solr/testCreateShardRepFactor_b_replica1/ has no replicas
   [junit4]   2> 1255095 INFO  (qtp1255938767-7420) [n:127.0.0.1:57886_solr c:testCreateShardRepFactor s:b  x:testCreateShardRepFactor_b_replica1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 1255100 INFO  (qtp844286241-7440) [n:127.0.0.1:57889_solr c:testCreateShardRepFactor s:a  x:testCreateShardRepFactor_a_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:57889/solr/testCreateShardRepFactor_a_replica1/ a
   [junit4]   2> 1255107 INFO  (qtp1255938767-7420) [n:127.0.0.1:57886_solr c:testCreateShardRepFactor s:b  x:testCreateShardRepFactor_b_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:57886/solr/testCreateShardRepFactor_b_replica1/ b
   [junit4]   2> 1255215 INFO  (zkCallback-1282-thread-1-processing-n:127.0.0.1:57889_solr) [n:127.0.0.1:57889_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCreateShardRepFactor/state.json] for collection [testCreateShardRepFactor] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1255217 INFO  (zkCallback-1277-thread-1-processing-n:127.0.0.1:57886_solr) [n:127.0.0.1:57886_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCreateShardRepFactor/state.json] for collection [testCreateShardRepFactor] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1255219 INFO  (qtp1255938767-7420) [n:127.0.0.1:57886_solr c:testCreateShardRepFactor s:b  x:testCreateShardRepFactor_b_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1255231 INFO  (qtp1255938767-7420) [n:127.0.0.1:57886_solr c:testCreateShardRepFactor s:b  x:testCreateShardRepFactor_b_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=testCreateShardRepFactor_b_replica1&action=CREATE&numShards=2&collection=testCreateShardRepFactor&shard=b&wt=javabin&version=2} status=0 QTime=1486
   [junit4]   2> 1255263 INFO  (qtp844286241-7440) [n:127.0.0.1:57889_solr c:testCreateShardRepFactor s:a  x:testCreateShardRepFactor_a_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1255268 INFO  (qtp844286241-7440) [n:127.0.0.1:57889_solr c:testCreateShardRepFactor s:a  x:testCreateShardRepFactor_a_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=testCreateShardRepFactor_a_replica1&action=CREATE&numShards=2&collection=testCreateShardRepFactor&shard=a&wt=javabin&version=2} status=0 QTime=1532
   [junit4]   2> 1255273 INFO  (qtp456925188-7414) [n:127.0.0.1:57887_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 1255378 INFO  (zkCallback-1282-thread-1-processing-n:127.0.0.1:57889_solr) [n:127.0.0.1:57889_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCreateShardRepFactor/state.json] for collection [testCreateShardRepFactor] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1255378 INFO  (zkCallback-1277-thread-1-processing-n:127.0.0.1:57886_solr) [n:127.0.0.1:57886_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCreateShardRepFactor/state.json] for collection [testCreateShardRepFactor] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1255596 INFO  (OverseerCollectionConfigSetProcessor-97643370076700680-127.0.0.1:57887_solr-n_0000000000) [n:127.0.0.1:57887_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1256279 INFO  (qtp456925188-7414) [n:127.0.0.1:57887_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={shards=a,b&replicationFactor=1&collection.configName=conf&name=testCreateShardRepFactor&router.name=implicit&action=CREATE&wt=javabin&version=2} status=0 QTime=2692
   [junit4]   2> 1256294 INFO  (qtp456925188-7415) [n:127.0.0.1:57887_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :createshard with params action=CREATESHARD&collection=testCreateShardRepFactor&shard=x&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1256301 INFO  (OverseerThreadFactory-3713-thread-2-processing-n:127.0.0.1:57887_solr) [n:127.0.0.1:57887_solr    ] o.a.s.c.CreateShardCmd Create shard invoked: {
   [junit4]   2>   "collection":"testCreateShardRepFactor",
   [junit4]   2>   "shard":"x",
   [junit4]   2>   "operation":"createshard"}
   [junit4]   2> 1256415 INFO  (zkCallback-1282-thread-1-processing-n:127.0.0.1:57889_solr) [n:127.0.0.1:57889_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCreateShardRepFactor/state.json] for collection [testCreateShardRepFactor] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1256415 INFO  (zkCallback-1277-thread-1-processing-n:127.0.0.1:57886_solr) [n:127.0.0.1:57886_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCreateShardRepFactor/state.json] for collection [testCreateShardRepFactor] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1256509 INFO  (OverseerThreadFactory-3713-thread-2-processing-n:127.0.0.1:57887_solr) [n:127.0.0.1:57887_solr    ] o.a.s.c.CreateShardCmd Creating shard testCreateShardRepFactor_x_replica1 as part of slice x of collection testCreateShardRepFactor on 127.0.0.1:57887_solr
   [junit4]   2> 1256515 INFO  (qtp456925188-7411) [n:127.0.0.1:57887_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&name=testCreateShardRepFactor_x_replica1&action=CREATE&numShards=1&collection=testCreateShardRepFactor&shard=x&wt=javabin&version=2
   [junit4]   2> 1256632 INFO  (zkCallback-1282-thread-1-processing-n:127.0.0.1:57889_solr) [n:127.0.0.1:57889_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCreateShardRepFactor/state.json] for collection [testCreateShardRepFactor] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1256632 INFO  (zkCallback-1277-thread-1-processing-n:127.0.0.1:57886_solr) [n:127.0.0.1:57886_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCreateShardRepFactor/state.json] for collection [testCreateShardRepFactor] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1256632 INFO  (zkCallback-1276-thread-1-processing-n:127.0.0.1:57887_solr) [n:127.0.0.1:57887_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCreateShardRepFactor/state.json] for collection [testCreateShardRepFactor] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1257551 INFO  (qtp456925188-7411) [n:127.0.0.1:57887_solr c:testCreateShardRepFactor s:x  x:testCreateShardRepFactor_x_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.6.0
   [junit4]   2> 1257563 INFO  (qtp456925188-7411) [n:127.0.0.1:57887_solr c:testCreateShardRepFactor s:x  x:testCreateShardRepFactor_x_replica1] o.a.s.s.IndexSchema [testCreateShardRepFactor_x_replica1] Schema name=cloud-dynamic
   [junit4]   2> 1257615 WARN  (qtp456925188-7411) [n:127.0.0.1:57887_solr c:testCreateShardRepFactor s:x  x:testCreateShardRepFactor_x_replica1] o.a.s.s.IndexSchema [testCreateShardRepFactor_x_replica1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 1257616 INFO  (qtp456925188-7411) [n:127.0.0.1:57887_solr c:testCreateShardRepFactor s:x  x:testCreateShardRepFactor_x_replica1] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
   [junit4]   2> 1257621 INFO  (qtp456925188-7411) [n:127.0.0.1:57887_solr c:testCreateShardRepFactor s:x  x:testCreateShardRepFactor_x_replica1] o.a.s.c.CoreContainer Creating SolrCore 'testCreateShardRepFactor_x_replica1' using configuration from collection testCreateShardRepFactor
   [junit4]   2> 1257622 INFO  (qtp456925188-7411) [n:127.0.0.1:57887_solr c:testCreateShardRepFactor s:x  x:testCreateShardRepFactor_x_replica1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1257622 INFO  (qtp456925188-7411) [n:127.0.0.1:57887_solr c:testCreateShardRepFactor s:x  x:testCreateShardRepFactor_x_replica1] o.a.s.c.SolrCore [[testCreateShardRepFactor_x_replica1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_98F8C321050165A8-001/tempDir-001/node3/testCreateShardRepFactor_x_replica1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_98F8C321050165A8-001/tempDir-001/node3/./testCreateShardRepFactor_x_replica1/data/]
   [junit4]   2> 1257776 INFO  (qtp456925188-7411) [n:127.0.0.1:57887_solr c:testCreateShardRepFactor s:x  x:testCreateShardRepFactor_x_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1257777 INFO  (qtp456925188-7411) [n:127.0.0.1:57887_solr c:testCreateShardRepFactor s:x  x:testCreateShardRepFactor_x_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1257780 INFO  (qtp456925188-7411) [n:127.0.0.1:57887_solr c:testCreateShardRepFactor s:x  x:testCreateShardRepFactor_x_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1257781 INFO  (qtp456925188-7411) [n:127.0.0.1:57887_solr c:testCreateShardRepFactor s:x  x:testCreateShardRepFactor_x_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1257786 INFO  (qtp456925188-7411) [n:127.0.0.1:57887_solr c:testCreateShardRepFactor s:x  x:testCreateShardRepFactor_x_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@369fd9b3[testCreateShardRepFactor_x_replica1] main]
   [junit4]   2> 1257789 INFO  (qtp456925188-7411) [n:127.0.0.1:57887_solr c:testCreateShardRepFactor s:x  x:testCreateShardRepFactor_x_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 1257792 INFO  (qtp456925188-7411) [n:127.0.0.1:57887_solr c:testCreateShardRepFactor s:x  x:testCreateShardRepFactor_x_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 1257792 INFO  (qtp456925188-7411) [n:127.0.0.1:57887_solr c:testCreateShardRepFactor s:x  x:testCreateShardRepFactor_x_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1257792 INFO  (searcherExecutor-3732-thread-1-processing-n:127.0.0.1:57887_solr x:testCreateShardRepFactor_x_replica1 s:x c:testCreateShardRepFactor) [n:127.0.0.1:57887_solr c:testCreateShardRepFactor s:x  x:testCreateShardRepFactor_x_replica1] o.a.s.c.SolrCore [testCreateShardRepFactor_x_replica1] Registered new searcher Searcher@369fd9b3[testCreateShardRepFactor_x_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1257792 INFO  (qtp456925188-7411) [n:127.0.0.1:57887_solr c:testCreateShardRepFactor s:x  x:testCreateShardRepFactor_x_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1562293926997524480
   [junit4]   2> 1257806 INFO  (qtp456925188-7411) [n:127.0.0.1:57887_solr c:testCreateShardRepFactor s:x  x:testCreateShardRepFactor_x_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1257807 INFO  (qtp456925188-7411) [n:127.0.0.1:57887_solr c:testCreateShardRepFactor s:x  x:testCreateShardRepFactor_x_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1257807 INFO  (qtp456925188-7411) [n:127.0.0.1:57887_solr c:testCreateShardRepFactor s:x  x:testCreateShardRepFactor_x_replica1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:57887/solr/testCreateShardRepFactor_x_replica1/
   [junit4]   2> 1257807 INFO  (qtp456925188-7411) [n:127.0.0.1:57887_solr c:testCreateShardRepFactor s:x  x:testCreateShardRepFactor_x_replica1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 1257807 INFO  (qtp456925188-7411) [n:127.0.0.1:57887_solr c:testCreateShardRepFactor s:x  x:testCreateShardRepFactor_x_replica1] o.a.s.c.SyncStrategy https://127.0.0.1:57887/solr/testCreateShardRepFactor_x_replica1/ has no replicas
   [junit4]   2> 1257807 INFO  (qtp456925188-7411) [n:127.0.0.1:57887_solr c:testCreateShardRepFactor s:x  x:testCreateShardRepFactor_x_replica1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 1257817 INFO  (qtp456925188-7411) [n:127.0.0.1:57887_solr c:testCreateShardRepFactor s:x  x:testCreateShardRepFactor_x_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:57887/solr/testCreateShardRepFactor_x_replica1/ x
   [junit4]   2> 1257924 INFO  (zkCallback-1277-thread-1-processing-n:127.0.0.1:57886_solr) [n:127.0.0.1:57886_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCreateShardRepFactor/state.json] for collection [testCreateShardRepFactor] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1257925 INFO  (zkCallback-1282-thread-1-processing-n:127.0.0.1:57889_solr) [n:127.0.0.1:57889_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCreateShardRepFactor/state.json] for collection [testCreateShardRepFactor] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1257925 INFO  (zkCallback-1276-thread-2-processing-n:127.0.0.1:57887_solr) [n:127.0.0.1:57887_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCreateShardRepFactor/state.json] for collection [testCreateShardRepFactor] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1257974 INFO  (qtp456925188-7411) [n:127.0.0.1:57887_solr c:testCreateShardRepFactor s:x  x:testCreateShardRepFactor_x_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1257983 INFO  (qtp456925188-7411) [n:127.0.0.1:57887_solr c:testCreateShardRepFactor s:x  x:testCreateShardRepFactor_x_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&name=testCreateShardRepFactor_x_replica1&action=CREATE&numShards=1&collection=testCreateShardRepFactor&shard=x&wt=javabin&version=2} status=0 QTime=1468
   [junit4]   2> 1257984 INFO  (OverseerThreadFactory-3713-thread-2-processing-n:127.0.0.1:57887_solr) [n:127.0.0.1:57887_solr    ] o.a.s.c.CreateShardCmd Finished create command on all shards for collection: testCreateShardRepFactor
   [junit4]   2> 1257990 INFO  (qtp456925188-7415) [n:127.0.0.1:57887_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CREATESHARD&collection=testCreateShardRepFactor&shard=x&wt=javabin&version=2} status=0 QTime=1695
   [junit4]   2> 1257992 INFO  (TEST-CustomCollectionTest.testCreateShardRepFactor-seed#[98F8C321050165A8]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testCreateShardRepFactor
   [junit4]   2> 1258054 INFO  (TEST-CustomCollectionTest.testRouteFieldForHashRouter-seed#[98F8C321050165A8]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testRouteFieldForHashRouter
   [junit4]   2> 1258056 INFO  (TEST-CustomCollectionTest.testRouteFieldForHashRouter-seed#[98F8C321050165A8]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 1258062 INFO  (qtp456925188-7412) [n:127.0.0.1:57887_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :delete with params name=testCreateShardRepFactor&action=DELETE&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1258067 INFO  (OverseerThreadFactory-3713-thread-3-processing-n:127.0.0.1:57887_solr) [n:127.0.0.1:57887_solr    ] o.a.s.c.OverseerCollectionMessageHandler Executing Collection Cmd : action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true
   [junit4]   2> 1258067 INFO  (OverseerCollectionConfigSetProcessor-97643370076700680-127.0.0.1:57887_solr-n_0000000000) [n:127.0.0.1:57887_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1258071 INFO  (qtp456925188-7413) [n:127.0.0.1:57887_solr    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.testCreateShardRepFactor.x.replica1
   [junit4]   2> 1258071 INFO  (qtp1255938767-7421) [n:127.0.0.1:57886_solr    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.testCreateShardRepFactor.b.replica1
   [junit4]   2> 1258072 INFO  (qtp456925188-7413) [n:127.0.0.1:57887_solr    ] o.a.s.c.SolrCore [testCreateShardRepFactor_x_replica1]  CLOSING SolrCore org.apache.solr.core.SolrCore@54bc9197
   [junit4]   2> 1258072 INFO  (qtp1255938767-7421) [n:127.0.0.1:57886_solr    ] o.a.s.c.SolrCore [testCreateShardRepFactor_b_replica1]  CLOSING SolrCore org.apache.solr.core.SolrCore@1ed241ef
   [junit4]   2> 1258072 INFO  (qtp844286241-7441) [n:127.0.0.1:57889_solr    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.testCreateShardRepFactor.a.replica1
   [junit4]   2> 1258072 INFO  (qtp844286241-7441) [n:127.0.0.1:57889_solr    ] o.a.s.c.SolrCore [testCreateShardRepFactor_a_replica1]  CLOSING SolrCore org.apache.solr.core.SolrCore@4550e6dc
   [junit4]   2> 1258075 INFO  (qtp844286241-7441) [n:127.0.0.1:57889_solr    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.testCreateShardRepFactor.a.replica1
   [junit4]   2> 1258076 INFO  (qtp456925188-7413) [n:127.0.0.1:57887_solr    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.testCreateShardRepFactor.x.replica1
   [junit4]   2> 1258076 INFO  (qtp1255938767-7421) [n:127.0.0.1:57886_solr    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.testCreateShardRepFactor.b.replica1
   [junit4]   2> 1258081 INFO  (qtp844286241-7441) [n:127.0.0.1:57889_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={deleteInstanceDir=true&core=testCreateShardRepFactor_a_replica1&qt=/admin/cores&deleteDataDir=true&action=UNLOAD&wt=javabin&version=2} status=0 QTime=10
   [junit4]   2> 1258092 INFO  (qtp456925188-7413) [n:127.0.0.1:57887_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={deleteInstanceDir=true&core=testCreateShardRepFactor_x_replica1&qt=/admin/cores&deleteDataDir=true&action=UNLOAD&wt=javabin&version=2} status=0 QTime=22
   [junit4]   2> 1258093 INFO  (qtp1255938767-7421) [n:127.0.0.1:57886_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={deleteInstanceDir=true&core=testCreateShardRepFactor_b_replica1&qt=/admin/cores&deleteDataDir=true&action=UNLOAD&wt=javabin&version=2} status=0 QTime=23
   [junit4]   2> 1258814 INFO  (qtp456925188-7412) [n:127.0.0.1:57887_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={name=testCreateShardRepFactor&action=DELETE&wt=javabin&version=2} status=0 QTime=752
   [junit4]   2> 1258817 INFO  (qtp456925188-7414) [n:127.0.0.1:57887_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=2&collection.configName=conf&maxShardsPerNode=3&router.field=shard_s&name=routeFieldColl&action=CREATE&numShards=4&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1258840 INFO  (OverseerThreadFactory-3713-thread-4-processing-n:127.0.0.1:57887_solr) [n:127.0.0.1:57887_solr    ] o.a.s.c.CreateCollectionCmd Create collection routeFieldColl
   [junit4]   2> 1258842 INFO  (OverseerCollectionConfigSetProcessor-97643370076700680-127.0.0.1:57887_solr-n_0000000000) [n:127.0.0.1:57887_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1259060 INFO  (qtp844286241-7436) [n:127.0.0.1:57889_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=routeFieldColl_shard2_replica1&action=CREATE&numShards=4&collection=routeFieldColl&shard=shard2&wt=javabin&version=2
   [junit4]   2> 1259063 INFO  (qtp1255938767-7407) [n:127.0.0.1:57886_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=routeFieldColl_shard3_replica1&action=CREATE&numShards=4&collection=routeFieldColl&shard=shard3&wt=javabin&version=2
   [junit4]   2> 1259069 INFO  (qtp456925188-7406) [n:127.0.0.1:57887_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=routeFieldColl_shard2_replica2&action=CREATE&numShards=4&collection=routeFieldColl&shard=shard2&wt=javabin&version=2
   [junit4]   2> 1259076 INFO  (qtp456925188-7411) [n:127.0.0.1:57887_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=routeFieldColl_shard4_replica2&action=CREATE&numShards=4&collection=routeFieldColl&shard=shard4&wt=javabin&version=2
   [junit4]   2> 1259076 INFO  (qtp1255938767-7422) [n:127.0.0.1:57886_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=routeFieldColl_shard1_replica1&action=CREATE&numShards=4&collection=routeFieldColl&shard=shard1&wt=javabin&version=2
   [junit4]   2> 1259079 INFO  (qtp844286241-7442) [n:127.0.0.1:57889_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=routeFieldColl_shard4_replica1&action=CREATE&numShards=4&collection=routeFieldColl&shard=shard4&wt=javabin&version=2
   [junit4]   2> 1259119 INFO  (qtp562650254-7432) [n:127.0.0.1:57888_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=routeFieldColl_shard3_replica2&action=CREATE&numShards=4&collection=routeFieldColl&shard=shard3&wt=javabin&version=2
   [junit4]   2> 1259152 INFO  (qtp562650254-7409) [n:127.0.0.1:57888_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=routeFieldColl_shard1_replica2&action=CREATE&numShards=4&collection=routeFieldColl&shard=shard1&wt=javabin&version=2
   [junit4]   2> 1259263 INFO  (zkCallback-1278-thread-1-processing-n:127.0.0.1:57888_solr) [n:127.0.0.1:57888_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/routeFieldColl/state.json] for collection [routeFieldColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1259264 INFO  (zkCallback-1277-thread-1-processing-n:127.0.0.1:57886_solr) [n:127.0.0.1:57886_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/routeFieldColl/state.json] for collection [routeFieldColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1259263 INFO  (zkCallback-1282-thread-1-processing-n:127.0.0.1:57889_solr) [n:127.0.0.1:57889_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/routeFieldColl/state.json] for collection [routeFieldColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1259266 INFO  (zkCallback-1276-thread-2-processing-n:127.0.0.1:57887_solr) [n:127.0.0.1:57887_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/routeFieldColl/state.json] for collection [routeFieldColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1260128 INFO  (qtp844286241-7442) [n:127.0.0.1:57889_solr c:routeFieldColl s:shard4  x:routeFieldColl_shard4_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.6.0
   [junit4]   2> 1260131 INFO  (qtp1255938767-7407) [n:127.0.0.1:57886_solr c:routeFieldColl s:shard3  x:routeFieldColl_shard3_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.6.0
   [junit4]   2> 1260168 INFO  (qtp844286241-7436) [n:127.0.0.1:57889_solr c:routeFieldColl s:shard2  x:routeFieldColl_shard2_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.6.0
   [junit4]   2> 1260179 INFO  (qtp456925188-7411) [n:127.0.0.1:57887_solr c:routeFieldColl s:shard4  x:routeFieldColl_shard4_replica2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.6.0
   [junit4]   2> 1260183 INFO  (qtp1255938767-7407) [n:127.0.0.1:57886_solr c:routeFieldColl s:shard3  x:routeFieldColl_shard3_replica1] o.a.s.s.IndexSchema [routeFieldColl_shard3_replica1] Schema name=cloud-dynamic
   [junit4]   2> 1260194 INFO  (qtp456925188-7411) [n:127.0.0.1:57887_solr c:routeFieldColl s:shard4  x:routeFieldColl_shard4_replica2] o.a.s.s.IndexSchema [routeFieldColl_shard4_replica2] Schema name=cloud-dynamic
   [junit4]   2> 1260214 INFO  (qtp562650254-7432) [n:127.0.0.1:57888_solr c:routeFieldColl s:shard3  x:routeFieldColl_shard3_replica2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.6.0
   [junit4]   2> 1260216 INFO  (qtp562650254-7409) [n:127.0.0.1:57888_solr c:routeFieldColl s:shard1  x:routeFieldColl_shard1_replica2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.6.0
   [junit4]   2> 1260226 INFO  (qtp1255938767-7422) [n:127.0.0.1:57886_solr c:routeFieldColl s:shard1  x:routeFieldColl_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.6.0
   [junit4]   2> 1260230 INFO  (qtp456925188-7406) [n:127.0.0.1:57887_solr c:routeFieldColl s:shard2  x:routeFieldColl_shard2_replica2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.6.0
   [junit4]   2> 1260230 INFO  (qtp562650254-7432) [n:127.0.0.1:57888_solr c:routeFieldColl s:shard3  x:routeFieldColl_shard3_replica2] o.a.s.s.IndexSchema [routeFieldColl_shard3_replica2] Schema name=cloud-dynamic
   [junit4]   2> 1260256 INFO  (qtp1255938767-7422) [n:127.0.0.1:57886_solr c:routeFieldColl s:shard1  x:routeFieldColl_shard1_replica1] o.a.s.s.IndexSchema [routeFieldColl_shard1_replica1] Schema name=cloud-dynamic
   [junit4]   2> 1260256 INFO  (qtp456925188-7406) [n:127.0.0.1:57887_solr c:routeFieldColl s:shard2  x:routeFieldColl_shard2_replica2] o.a.s.s.IndexSchema [routeFieldColl_shard2_replica2] Schema name=cloud-dynamic
   [junit4]   2> 1260267 INFO  (qtp844286241-7442) [n:127.0.0.1:57889_solr c:routeFieldColl s:shard4  x:routeFieldColl_shard4_replica1] o.a.s.s.IndexSchema [routeFieldColl_shard4_replica1] Schema name=cloud-dynamic
   [junit4]   2> 1260283 INFO  (qtp844286241-7436) [n:127.0.0.1:57889_solr c:routeFieldColl s:shard2  x:routeFieldColl_shard2_replica1] o.a.s.s.IndexSchema [routeFieldColl_shard2_replica1] Schema name=cloud-dynamic
   [junit4]   2> 1260315 INFO  (qtp562650254-7409) [n:127.0.0.1:57888_solr c:routeFieldColl s:shard1  x:routeFieldColl_shard1_replica2] o.a.s.s.IndexSchema [routeFieldColl_shard1_replica2] Schema name=cloud-dynamic
   [junit4]   2> 1260349 WARN  (qtp844286241-7442) [n:127.0.0.1:57889_solr c:routeFieldColl s:shard4  x:routeFieldColl_shard4_replica1] o.a.s.s.IndexSchema [routeFieldColl_shard4_replica1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 1260349 INFO  (qtp844286241-7442) [n:127.0.0.1:57889_solr c:routeFieldColl s:shard4  x:routeFieldColl_shard4_replica1] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
   [junit4]   2> 1260355 INFO  (qtp844286241-7442) [n:127.0.0.1:57889_solr c:routeFieldColl s:shard4  x:routeFieldColl_shard4_replica1] o.a.s.c.CoreContainer Creating SolrCore 'routeFieldColl_shard4_replica1' using configuration from collection routeFieldColl
   [junit4]   2> 1260356 INFO  (qtp844286241-7442) [n:127.0.0.1:57889_solr c:routeFieldColl s:shard4  x:routeFieldColl_shard4_replica1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1260357 INFO  (qtp844286241-7442) [n:127.0.0.1:57889_solr c:routeFieldColl s:shard4  x:routeFieldColl_shard4_replica1] o.a.s.c.SolrCore [[routeFieldColl_shard4_replica1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_98F8C321050165A8-001/tempDir-001/node4/routeFieldColl_shard4_replica1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_98F8C321050165A8-001/tempDir-001/node4/./routeFieldColl_shard4_replica1/data/]
   [junit4]   2> 1260360 WARN  (qtp844286241-7436) [n:127.0.0.1:57889_solr c:routeFieldColl s:shard2  x:routeFieldColl_shard2_replica1] o.a.s.s.IndexSchema [routeFieldColl_shard2_replica1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 1260360 WARN  (qtp456925188-7406) [n:127.0.0.1:57887_solr c:routeFieldColl s:shard2  x:routeFieldColl_shard2_replica2] o.a.s.s.IndexSchema [routeFieldColl_shard2_replica2] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 1260361 INFO  (qtp844286241-7436) [n:127.0.0.1:57889_solr c:routeFieldColl s:shard2  x:routeFieldColl_shard2_replica1] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
   [junit4]   2> 1260361 INFO  (qtp456925188-7406) [n:127.0.0.1:57887_solr c:routeFieldColl s:shard2  x:routeFieldColl_shard2_replica2] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
   [junit4]   2> 1260362 WARN  (qtp562650254-7432) [n:127.0.0.1:57888_solr c:routeFieldColl s:shard3  x:routeFieldColl_shard3_replica2] o.a.s.s.IndexSchema [routeFieldColl_shard3_replica2] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 1260372 INFO  (qtp562650254-7432) [n:127.0.0.1:57888_solr c:routeFieldColl s:shard3  x:routeFieldColl_shard3_replica2] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
   [junit4]   2> 1260371 WARN  (qtp1255938767-7422) [n:127.0.0.1:57886_solr c:routeFieldColl s:shard1  x:routeFieldColl_shard1_replica1] o.a.s.s.IndexSchema [routeFieldColl_shard1_replica1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 1260374 INFO  (qtp1255938767-7422) [n:127.0.0.1:57886_solr c:routeFieldColl s:shard1  x:routeFieldColl_shard1_replica1] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
   [junit4]   2> 1260374 WARN  (qtp562650254-7409) [n:127.0.0.1:57888_solr c:routeFieldColl s:shard1  x:routeFieldColl_shard1_replica2] o.a.s.s.IndexSchema [routeFieldColl_shard1_replica2] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 1260363 WARN  (qtp456925188-7411) [n:127.0.0.1:57887_solr c:routeFieldColl s:shard4  x:routeFieldColl_shard4_replica2] o.a.s.s.IndexSchema [routeFieldColl_shard4_replica2] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 1260375 INFO  (qtp456925188-7411) [n:127.0.0.1:57887_solr c:routeFieldColl s:shard4  x:routeFieldColl_shard4_replica2] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
   [junit4]   2> 1260375 INFO  (qtp1255938767-7422) [n:127.0.0.1:57886_solr c:routeFieldColl s:shard1  x:routeFieldColl_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'routeFieldColl_shard1_replica1' using configuration from collection routeFieldColl
   [junit4]   2> 1260376 INFO  (qtp1255938767-7422) [n:127.0.0.1:57886_solr c:routeFieldColl s:shard1  x:routeFieldColl_shard1_replica1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1260376 INFO  (qtp456925188-7411) [n:127.0.0.1:57887_solr c:routeFieldColl s:shard4  x:routeFieldColl_shard4_replica2] o.a.s.c.CoreContainer Creating SolrCore 'routeFieldColl_shard4_replica2' using configuration from collection routeFieldColl
   [junit4]   2> 1260376 INFO  (qtp1255938767-7422) [n:127.0.0.1:57886_solr c:routeFieldColl s:shard1  x:routeFieldColl_shard1_replica1] o.a.s.c.SolrCore [[routeFieldColl_shard1_replica1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_98F8C321050165A8-001/tempDir-001/node1/routeFieldColl_shard1_replica1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_98F8C321050165A8-001/tempDir-001/node1/./routeFieldColl_shard1_replica1/data/]
   [junit4]   2> 1260377 INFO  (qtp456925188-7411) [n:127.0.0.1:57887_solr c:routeFieldColl s:shard4  x:routeFieldColl_shard4_replica2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1260377 INFO  (qtp456925188-7411) [n:127.0.0.1:57887_solr c:routeFieldColl s:shard4  x:routeFieldColl_shard4_replica2] o.a.s.c.SolrCore [[routeFieldColl_shard4_replica2] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_98F8C321050165A8-001/tempDir-001/node3/routeFieldColl_shard4_replica2], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_98F8C321050165A8-001/tempDir-001/node3/./routeFieldColl_shard4_replica2/data/]
   [junit4]   2> 1260378 INFO  (qtp562650254-7409) [n:127.0.0.1:57888_solr c:routeFieldColl s:shard1  x:routeFieldColl_shard1_replica2] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
   [junit4]   2> 1260368 INFO  (qtp844286241-7436) [n:127.0.0.1:57889_solr c:routeFieldColl s:shard2  x:routeFieldColl_shard2_replica1] o.a.s.c.CoreContainer Creating SolrCore 'routeFieldColl_shard2_replica1' using configuration from collection routeFieldColl
   [junit4]   2> 1260380 INFO  (qtp844286241-7436) [n:127.0.0.1:57889_solr c:routeFieldColl s:shard2  x:routeFieldColl_shard2_replica1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1260380 INFO  (qtp844286241-7436) [n:127.0.0.1:57889_solr c:routeFieldColl s:shard2  x:routeFieldColl_shard2_replica1] o.a.s.c.SolrCore [[routeFieldColl_shard2_replica1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_98F8C321050165A8-001/tempDir-001/node4/routeFieldColl_shard2_replica1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_98F8C321050165A8-001/tempDir-001/node4/./routeFieldColl_shard2_replica1/data/]
   [junit4]   2> 1260381 INFO  (qtp562650254-7432) [n:127.0.0.1:57888_solr c:routeFieldColl s:shard3  x:routeFieldColl_shard3_replica2] o.a.s.c.CoreContainer Creating SolrCore 'routeFieldColl_shard3_replica2' using configuration from collection routeFieldColl
   [junit4]   2> 1260381 INFO  (qtp562650254-7432) [n:127.0.0.1:57888_solr c:routeFieldColl s:shard3  x:routeFieldColl_shard3_replica2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1260366 WARN  (qtp1255938767-7407) [n:127.0.0.1:57886_solr c:routeFieldColl s:shard3  x:routeFieldColl_shard3_replica1] o.a.s.s.IndexSchema [routeFieldColl_shard3_replica1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 1260381 INFO  (qtp562650254-7432) [n:127.0.0.1:57888_solr c:routeFieldColl s:shard3  x:routeFieldColl_shard3_replica2] o.a.s.c.SolrCore [[routeFieldColl_shard3_replica2] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_98F8C321050165A8-001/tempDir-001/node2/routeFieldColl_shard3_replica2], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_98F8C321050165A8-001/tempDir-001/node2/./routeFieldColl_shard3_replica2/data/]
   [junit4]   2> 1260382 INFO  (qtp1255938767-7407) [n:127.0.0.1:57886_solr c:routeFieldColl s:shard3  x:routeFieldColl_shard3_replica1] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
   [junit4]   2> 1260365 INFO  (qtp456925188-7406) [n:127.0.0.1:57887_solr c:routeFieldColl s:shard2  x:routeFieldColl_shard2_replica2] o.a.s.c.CoreContainer Creating SolrCore 'routeFieldColl_shard2_replica2' using configuration from collection routeFieldColl
   [junit4]   2> 1260383 INFO  (qtp456925188-7406) [n:127.0.0.1:57887_solr c:routeFieldColl s:shard2  x:routeFieldColl_shard2_replica2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1260383 INFO  (qtp1255938767-7407) [n:127.0.0.1:57886_solr c:routeFieldColl s:shard3  x:routeFieldColl_shard3_replica1] o.a.s.c.CoreContainer Creating SolrCore 'routeFieldColl_shard3_replica1' using configuration from collection routeFieldColl
   [junit4]   2> 1260383 INFO  (qtp456925188-7406) [n:127.0.0.1:57887_solr c:routeFieldColl s:shard2  x:routeFieldColl_shard2_replica2] o.a.s.c.SolrCore [[routeFieldColl_shard2_replica2] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_98F8C321050165A8-001/tempDir-001/node3/routeFieldColl_shard2_replica2], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_98F8C321050165A8-001/tempDir-001/node3/./routeFieldColl_shard2_replica2/data/]
   [junit4]   2> 1260384 INFO  (qtp1255938767-7407) [n:127.0.0.1:57886_solr c:routeFieldColl s:shard3  x:routeFieldColl_shard3_replica1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1260385 INFO  (qtp1255938767-7407) [n:127.0.0.1:57886_solr c:routeFieldColl s:shard3  x:routeFieldColl_shard3_replica1] o.a.s.c.SolrCore [[routeFieldColl_shard3_replica1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_98F8C321050165A8-001/tempDir-001/node1/routeFieldColl_shard3_replica1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_98F8C321050165A8-001/tempDir-001/node1/./routeFieldColl_shard3_replica1/data/]
   [junit4]   2> 1260386 INFO  (qtp562650254-7409) [n:127.0.0.1:57888_solr c:routeFieldColl s:shard1  x:routeFieldColl_shard1_replica2] o.a.s.c.CoreContainer Creating SolrCore 'routeFieldColl_shard1_replica2' using configuration from collection routeFieldColl
   [junit4]   2> 1260387 INFO  (qtp562650254-7409) [n:127.0.0.1:57888_solr c:routeFieldColl s:shard1  x:routeFieldColl_shard1_replica2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1260387 INFO  (qtp562650254-7409) [n:127.0.0.1:57888_solr c:routeFieldColl s:shard1  x:routeFieldColl_shard1_replica2] o.a.s.c.SolrCore [[routeFieldColl_shard1_replica2] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_98F8C321050165A8-001/tempDir-001/node2/routeFieldColl_shard1_replica2], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_98F8C321050165A8-001/tempDir-001/node2/./routeFieldColl_shard1_replica2/data/]
   [junit4]   2> 1260696 INFO  (qtp844286241-7436) [n:127.0.0.1:57889_solr c:routeFieldColl s:shard2  x:routeFieldColl_shard2_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1260696 INFO  (qtp844286241-7436) [n:127.0.0.1:57889_solr c:routeFieldColl s:shard2  x:routeFieldColl_shard2_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1260699 INFO  (qtp1255938767-7407) [n:127.0.0.1:57886_solr c:routeFieldColl s:shard3  x:routeFieldColl_shard3_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1260699 INFO  (qtp1255938767-7407) [n:127.0.0.1:57886_solr c:routeFieldColl s:shard3  x:routeFieldColl_shard3_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1260704 INFO  (qtp1255938767-7407) [n:127.0.0.1:57886_solr c:routeFieldColl s:shard3  x:routeFieldColl_shard3_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1260705 INFO  (qtp1255938767-7407) [n:127.0.0.1:57886_solr c:routeFieldColl s:shard3  x:routeFieldColl_shard3_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1260715 INFO  (qtp844286241-7442) [n:127.0.0.1:57889_solr c:routeFieldColl s:shard4  x:routeFieldColl_shard4_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1260715 INFO  (qtp844286241-7442) [n:127.0.0.1:57889_solr c:routeFieldColl s:shard4  x:routeFieldColl_shard4_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1260716 INFO  (qtp844286241-7436) [n:127.0.0.1:57889_solr c:routeFieldColl s:shard2  x:routeFieldColl_shard2_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1260716 INFO  (qtp844286241-7436) [n:127.0.0.1:57889_solr c:routeFieldColl s:shard2  x:routeFieldColl_shard2_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1260718 INFO  (qtp844286241-7442) [n:127.0.0.1:57889_solr c:routeFieldColl s:shard4  x:routeFieldColl_shard4_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1260718 INFO  (qtp844286241-7442) [n:127.0.0.1:57889_solr c:routeFieldColl s:shard4  x:routeFieldColl_shard4_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1260728 INFO  (qtp562650254-7432) [n:127.0.0.1:57888_solr c:routeFieldColl s:shard3  x:routeFieldColl_shard3_replica2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1260728 INFO  (qtp562650254-7432) [n:127.0.0.1:57888_solr c:routeFieldColl s:shard3  x:routeFieldColl_shard3_replica2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1260729 INFO  (qtp456925188-7406) [n:127.0.0.1:57887_solr c:routeFieldColl s:shard2  x:routeFieldColl_shard2_replica2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1260729 INFO  (qtp456925188-7406) [n:127.0.0.1:57887_solr c:routeFieldColl s:shard2  x:routeFieldColl_shard2_replica2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1260730 INFO  (qtp456925188-7411) [n:127.0.0.1:57887_solr c:routeFieldColl s:shard4  x:routeFieldColl_shard4_replica2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1260730 INFO  (qtp456925188-7411) [n:127.0.0.1:57887_solr c:routeFieldColl s:shard4  x:routeFieldColl_shard4_replica2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1260732 INFO  (qtp1255938767-7422) [n:127.0.0.1:57886_solr c:routeFieldColl s:shard1  x:routeFieldColl_shard1_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1260732 INFO  (qtp1255938767-7422) [n:127.0.0.1:57886_solr c:routeFieldColl s:shard1  x:routeFieldColl_shard1_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1260732 INFO  (qtp562650254-7432) [n:127.0.0.1:57888_solr c:routeFieldColl s:shard3  x:routeFieldColl_shard3_replica2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1260732 INFO  (qtp562650254-7432) [n:127.0.0.1:57888_solr c:routeFieldColl s:shard3  x:routeFieldColl_shard3_replica2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1260733 INFO  (qtp456925188-7411) [n:127.0.0.1:57887_solr c:routeFieldColl s:shard4  x:routeFieldColl_shard4_replica2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1260733 INFO  (qtp456925188-7411) [n:127.0.0.1:57887_solr c:routeFieldColl s:shard4  x:routeFieldColl_shard4_replica2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1260734 INFO  (qtp1255938767-7422) [n:127.0.0.1:57886_solr c:routeFieldColl s:shard1  x:routeFieldColl_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1260735 INFO  (qtp1255938767-7422) [n:127.0.0.1:57886_solr c:routeFieldColl s:shard1  x:routeFieldColl_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1260739 INFO  (qtp456925188-7406) [n:127.0.0.1:57887_solr c:routeFieldColl s:shard2  x:routeFieldColl_shard2_replica2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1260739 INFO  (qtp456925188-7406) [n:127.0.0.1:57887_solr c:routeFieldColl s:shard2  x:routeFieldColl_shard2_replica2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1260742 INFO  (qtp1255938767-7407) [n:127.0.0.1:57886_solr c:routeFieldColl s:shard3  x:routeFieldColl_shard3_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@16dd2c6e[routeFieldColl_shard3_replica1] main]
   [junit4]   2> 1260745 INFO  (qtp1255938767-7407) [n:127.0.0.1:57886_solr c:routeFieldColl s:shard3  x:routeFieldColl_shard3_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 1260746 INFO  (qtp1255938767-7407) [n:127.0.0.1:57886_solr c:routeFieldColl s:shard3  x:routeFieldColl_shard3_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 1260746 INFO  (qtp1255938767-7407) [n:127.0.0.1:57886_solr c:routeFieldColl s:shard3  x:routeFieldColl_shard3_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1260747 INFO  (searcherExecutor-3743-thread-1-processing-n:127.0.0.1:57886_solr x:routeFieldColl_shard3_replica1 s:shard3 c:routeFieldColl) [n:127.0.0.1:57886_solr c:routeFieldColl s:shard3  x:routeFieldColl_shard3_replica1] o.a.s.c.SolrCore [routeFieldColl_shard3_replica1] Registered new searcher Searcher@16dd2c6e[routeFieldColl_shard3_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1260747 INFO  (qtp1255938767-7407) [n:127.0.0.1:57886_solr c:routeFieldColl s:shard3  x:routeFieldColl_shard3_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1562293930096066560
   [junit4]   2> 1260748 INFO  (qtp562650254-7409) [n:127.0.0.1:57888_solr c:routeFieldColl s:shard1  x:routeFieldColl_shard1_replica2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1260749 INFO  (qtp562650254-7409) [n:127.0.0.1:57888_solr c:routeFieldColl s:shard1  x:routeFieldColl_shard1_replica2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1260749 INFO  (qtp844286241-7436) [n:127.0.0.1:57889_solr c:routeFieldColl s:shard2  x:routeFieldColl_shard2_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@16e05752[routeFieldColl_shard2_replica1] main]
   [junit4]   2> 1260752 INFO  (qtp844286241-7436) [n:127.0.0.1:57889_solr c:routeFieldColl s:shard2  x:routeFieldColl_shard2_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 1260753 INFO  (qtp844286241-7436) [n:127.0.0.1:57889_solr c:routeFieldColl s:shard2  x:routeFieldColl_shard2_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 1260753 INFO  (qtp844286241-7436) [n:127.0.0.1:57889_solr c:routeFieldColl s:shard2  x:routeFieldColl_shard2_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1260753 INFO  (qtp562650254-7409) [n:127.0.0.1:57888_solr c:routeFieldColl s:shard1  x:routeFieldColl_shard1_replica2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1260753 INFO  (qtp562650254-7409) [n:127.0.0.1:57888_solr c:routeFieldColl s:shard1  x:routeFieldColl_shard1_replica2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1260753 INFO  (searcherExecutor-3740-thread-1-processing-n:127.0.0.1:57889_solr x:routeFieldColl_shard2_replica1 s:shard2 c:routeFieldColl) [n:127.0.0.1:57889_solr c:routeFieldColl s:shard2  x:routeFieldColl_shard2_replica1] o.a.s.c.SolrCore [routeFieldColl_shard2_replica1] Registered new searcher Searcher@16e05752[routeFieldColl_shard2_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1260753 INFO  (qtp844286241-7436) [n:127.0.0.1:57889_solr c:routeFieldColl s:shard2  x:routeFieldColl_shard2_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1562293930102358016
   [junit4]   2> 1260758 INFO  (qtp844286241-7442) [n:127.0.0.1:57889_solr c:routeFieldColl s:shard4  x:routeFieldColl_shard4_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@1a90d822[routeFieldColl_shard4_replica1] main]
   [junit4]   2> 1260760 INFO  (qtp1255938767-7407) [n:127.0.0.1:57886_solr c:routeFieldColl s:shard3  x:routeFieldColl_shard3_replica1] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard3: total=2 found=1 timeoutin=9998ms
   [junit4]   2> 1260762 INFO  (qtp844286241-7442) [n:127.0.0.1:57889_solr c:routeFieldColl s:shard4  x:routeFieldColl_shard4_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 1260763 INFO  (qtp844286241-7442) [n:127.0.0.1:57889_solr c:routeFieldColl s:shard4  x:routeFieldColl_shard4_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 1260763 INFO  (qtp844286241-7442) [n:127.0.0.1:57889_solr c:routeFieldColl s:shard4  x:routeFieldColl_shard4_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1260764 INFO  (searcherExecutor-3737-thread-1-processing-n:127.0.0.1:57889_solr x:routeFieldColl_shard4_replica1 s:shard4 c:routeFieldColl) [n:127.0.0.1:57889_solr c:routeFieldColl s:shard4  x:routeFieldColl_shard4_replica1] o.a.s.c.SolrCore [routeFieldColl_shard4_replica1] Registered new searcher Searcher@1a90d822[routeFieldColl_shard4_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1260764 INFO  (qtp844286241-7442) [n:127.0.0.1:57889_solr c:routeFieldColl s:shard4  x:routeFieldColl_shard4_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1562293930113892352
   [junit4]   2> 1260764 INFO  (qtp562650254-7432) [n:127.0.0.1:57888_solr c:routeFieldColl s:shard3  x:routeFieldColl_shard3_replica2] o.a.s.s.SolrIndexSearcher Opening [Searcher@507f5df7[routeFieldColl_shard3_replica2] main]
   [junit4]   2> 1260766 INFO  (qtp456925188-7411) [n:127.0.0.1:57887_solr c:routeFieldColl s:shard4  x:routeFieldColl_shard4_replica2] o.a.s.s.SolrIndexSearcher Opening [Searcher@71debf9e[routeFieldColl_shard4_replica2] main]
   [junit4]   2> 1260767 INFO  (qtp562650254-7432) [n:127.0.0.1:57888_solr c:routeFieldColl s:shard3  x:routeFieldColl_shard3_replica2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 1260768 INFO  (qtp1255938767-7422) [n:127.0.0.1:57886_solr c:routeFieldColl s:shard1  x:routeFieldColl_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@628165d0[routeFieldColl_shard1_replica1] main]
   [junit4]   2> 1260768 INFO  (qtp562650254-7432) [n:127.0.0.1:57888_solr c:routeFieldColl s:shard3  x:routeFieldColl_shard3_replica2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 1260770 INFO  (qtp562650254-7432) [n:127.0.0.1:57888_solr c:routeFieldColl s:shard3  x:routeFieldColl_shard3_replica2] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1260772 INFO  (zkCallback-1278-thread-1-processing-n:127.0.0.1:

[...truncated too long message...]

3
   [junit4]   2> 1276783 WARN  (zkCallback-1282-thread-1-processing-n:127.0.0.1:57889_solr) [n:127.0.0.1:57889_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 1276784 INFO  (jetty-closer-1261-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> 1276784 INFO  (jetty-closer-1261-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@73915db5{/solr,null,UNAVAILABLE}
   [junit4]   2> 1276786 INFO  (zkCallback-1278-thread-2-processing-n:127.0.0.1:57888_solr) [n:127.0.0.1:57888_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
   [junit4]   2> 1276793 INFO  (zkCallback-1276-thread-2-processing-n:127.0.0.1:57887_solr) [n:127.0.0.1:57887_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
   [junit4]   2> 1276793 INFO  (zkCallback-1277-thread-3-processing-n:127.0.0.1:57886_solr) [n:127.0.0.1:57886_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
   [junit4]   2> 1276797 INFO  (coreCloseExecutor-3883-thread-2) [n:127.0.0.1:57886_solr c:implicitcoll s:c r:core_node11 x:implicitcoll_c_replica4] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.implicitcoll.c.replica4
   [junit4]   2> 1276811 WARN  (zkCallback-1278-thread-2-processing-n:127.0.0.1:57888_solr) [n:127.0.0.1:57888_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 1276811 INFO  (jetty-closer-1261-thread-2) [    ] o.a.s.c.Overseer Overseer (id=97643370076700680-127.0.0.1:57887_solr-n_0000000000) closing
   [junit4]   2> 1276812 INFO  (jetty-closer-1261-thread-3) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> 1276812 INFO  (OverseerStateUpdate-97643370076700680-127.0.0.1:57887_solr-n_0000000000) [n:127.0.0.1:57887_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:57887_solr
   [junit4]   2> 1276812 INFO  (jetty-closer-1261-thread-3) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@60c4c8f2{/solr,null,UNAVAILABLE}
   [junit4]   2> 1276813 INFO  (zkCallback-1277-thread-3-processing-n:127.0.0.1:57886_solr) [n:127.0.0.1:57886_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (2)
   [junit4]   2> 1276814 INFO  (zkCallback-1276-thread-2-processing-n:127.0.0.1:57887_solr) [n:127.0.0.1:57887_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (2)
   [junit4]   2> 1276818 INFO  (zkCallback-1277-thread-2-processing-n:127.0.0.1:57886_solr) [n:127.0.0.1:57886_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
   [junit4]   2> 1276818 INFO  (zkCallback-1277-thread-1-processing-n:127.0.0.1:57886_solr) [n:127.0.0.1:57886_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:57886_solr
   [junit4]   2> 1276824 WARN  (zkCallback-1277-thread-1-processing-n:127.0.0.1:57886_solr) [n:127.0.0.1:57886_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 1276824 INFO  (jetty-closer-1261-thread-4) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> 1276825 INFO  (jetty-closer-1261-thread-4) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@17f985db{/solr,null,UNAVAILABLE}
   [junit4]   2> 1278324 WARN  (zkCallback-1276-thread-2-processing-n:127.0.0.1:57887_solr) [n:127.0.0.1:57887_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 1278325 INFO  (jetty-closer-1261-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> 1278325 INFO  (jetty-closer-1261-thread-2) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@710f38a{/solr,null,UNAVAILABLE}
   [junit4]   2> 1278329 INFO  (SUITE-CustomCollectionTest-seed#[98F8C321050165A8]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:57884 57884
   [junit4]   2> 1278354 INFO  (Thread-1756) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:57884 57884
   [junit4]   2> 1281445 WARN  (Thread-1756) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	15	/solr/configs/conf
   [junit4]   2> 	9	/solr/aliases.json
   [junit4]   2> 	4	/solr/security.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	9	/solr/clusterprops.json
   [junit4]   2> 	9	/solr/clusterstate.json
   [junit4]   2> 	4	/solr/collections/testCreateShardRepFactor/state.json
   [junit4]   2> 	4	/solr/collections/implicitcoll/state.json
   [junit4]   2> 	4	/solr/collections/routeFieldColl/state.json
   [junit4]   2> 	4	/solr/collections/withShardField/state.json
   [junit4]   2> 	2	/solr/collections/implicitcoll/leader_elect/c/election/97643370076700680-core_node6-n_0000000000
   [junit4]   2> 	2	/solr/collections/implicitcoll/leader_elect/a/election/97643370076700682-core_node9-n_0000000000
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	74	/solr/overseer/queue
   [junit4]   2> 	28	/solr/collections
   [junit4]   2> 	9	/solr/live_nodes
   [junit4]   2> 	9	/solr/overseer/collection-queue-work
   [junit4]   2> 	6	/solr/overseer/queue-work
   [junit4]   2> 
   [junit4]   2> 1281445 INFO  (SUITE-CustomCollectionTest-seed#[98F8C321050165A8]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_98F8C321050165A8-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene62): {shard_s=PostingsFormat(name=LuceneVarGapFixedInterval), _version_=PostingsFormat(name=LuceneVarGapFixedInterval), id=Lucene50(blocksize=128)}, docValues:{}, maxPointsInLeafNode=1754, maxMBSortInHeap=7.856819922704462, sim=RandomSimilarity(queryNorm=false,coord=no): {}, locale=en-IN, timezone=America/Glace_Bay
   [junit4]   2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_121 (64-bit)/cpus=3,threads=1,free=179654024,total=448266240
   [junit4]   2> NOTE: All tests run in this JVM: [HighlighterTest, SaslZkACLProviderTest, ZkControllerTest, PeerSyncReplicationTest, ExitableDirectoryReaderTest, SpatialFilterTest, HdfsRecoveryZkTest, SpatialHeatmapFacetsTest, CoreAdminRequestStatusTest, TestEmbeddedSolrServerConstructors, TestGeoJSONResponseWriter, TestTolerantUpdateProcessorCloud, SolrRequestParserTest, SimplePostToolTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, TestDelegationWithHadoopAuth, VersionInfoTest, TestStressVersions, PeerSyncWithIndexFingerprintCachingTest, AtomicUpdatesTest, TestDistributedSearch, AddBlockUpdateTest, SystemInfoHandlerTest, TestReplicationHandlerBackup, TestHighlightDedupGrouping, MultiThreadedOCPTest, TestBinaryField, TemplateUpdateProcessorTest, SchemaVersionSpecificBehaviorTest, TestUseDocValuesAsStored, TestSQLHandlerNonCloud, LeaderInitiatedRecoveryOnCommitTest, TestHashPartitioner, OpenExchangeRatesOrgProviderTest, DocValuesTest, TestManagedSchema, NumericFieldsTest, TestSolrQueryResponse, StandardRequestHandlerTest, CloneFieldUpdateProcessorFactoryTest, HdfsTlogReplayBufferedWhileIndexingTest, TestFieldCacheVsDocValues, SolrMetricsIntegrationTest, DistributedFacetPivotLargeTest, TestComponentsName, TestExactSharedStatsCache, ZkCLITest, ClusterStateUpdateTest, TestManagedResourceStorage, ClassificationUpdateProcessorIntegrationTest, TermsComponentTest, TestNoOpRegenerator, SubstringBytesRefFilterTest, TestSerializedLuceneMatchVersion, OutputWriterTest, SoftAutoCommitTest, TestExtendedDismaxParser, DOMUtilTest, TestCharFilters, ChangedSchemaMergeTest, IndexBasedSpellCheckerTest, TestAnalyzedSuggestions, CdcrRequestHandlerTest, AssignTest, TestFileDictionaryLookup, TestClassicSimilarityFactory, SolrTestCaseJ4Test, DistanceUnitsTest, TestWordDelimiterFilterFactory, DistributedFacetPivotSmallAdvancedTest, AlternateDirectoryTest, BadComponentTest, TestClusterProperties, DirectSolrSpellCheckerTest, TestAnalyzeInfixSuggestions, TestSolrCLIRunExample, HighlighterMaxOffsetTest, TestJavabinTupleStreamParser, TestFieldSortValues, TestSolrIndexConfig, TestConfigSetProperties, TestJettySolrRunner, TestComplexPhraseQParserPlugin, TestManagedSchemaThreadSafety, TestCloudSchemaless, AnalysisAfterCoreReloadTest, SolrMetricManagerTest, TestCloudPseudoReturnFields, DeleteLastCustomShardedReplicaTest, TestRandomDVFaceting, TestDownShardTolerantSearch, CloudExitableDirectoryReaderTest, BigEndianAscendingWordSerializerTest, TestIntervalFaceting, PolyFieldTest, TestSolrDynamicMBean, SparseHLLTest, QueryEqualityTest, TestDefaultStatsCache, DocumentAnalysisRequestHandlerTest, TestRuleBasedAuthorizationPlugin, LeaderElectionTest, TestStressInPlaceUpdates, DeleteReplicaTest, RequestLoggingTest, SolrCloudExampleTest, DistribDocExpirationUpdateProcessorTest, SuggesterFSTTest, FileUtilsTest, TestDocTermOrds, TestSolrDeletionPolicy1, TestShardHandlerFactory, ReplicaListTransformerTest, TestSolrQueryParserResource, TestDocumentBuilder, DocValuesMultiTest, PrimUtilsTest, TestStandardQParsers, SolrCoreTest, TestDynamicFieldCollectionResource, TestBinaryResponseWriter, TestTolerantSearch, TestSolrCloudSnapshots, TestLeaderElectionZkExpiry, TestLegacyFieldCache, TestSolrCloudWithSecureImpersonation, DocValuesNotIndexedTest, SolrSlf4jReporterTest, CustomCollectionTest]
   [junit4] Completed [270/700 (1!)] on J0 in 32.12s, 4 tests, 1 error <<< FAILURES!

[...truncated 64161 lines...]


[JENKINS] Lucene-Solr-6.x-MacOSX (64bit/jdk1.8.0) - Build # 771 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-6.x-MacOSX/771/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseSerialGC

35 tests failed.
FAILED:  org.apache.solr.cloud.TestDeleteCollectionOnDownNodes.deleteCollectionWithDownNodes

Error Message:
Timed out waiting for leader elections null Last available state: null

Stack Trace:
java.lang.AssertionError: Timed out waiting for leader elections
null
Last available state: null
	at __randomizedtesting.SeedInfo.seed([E1B1817A22BABE1A:7F84E5820499F292]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:265)
	at org.apache.solr.cloud.TestDeleteCollectionOnDownNodes.deleteCollectionWithDownNodes(TestDeleteCollectionOnDownNodes.java:47)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry

Error Message:


Stack Trace:
java.lang.NullPointerException
	at org.apache.zookeeper.server.NIOServerCnxnFactory.getLocalPort(NIOServerCnxnFactory.java:134)
	at org.apache.solr.cloud.ZkTestServer$ZKServerMain.shutdown(ZkTestServer.java:335)
	at org.apache.solr.cloud.ZkTestServer.shutdown(ZkTestServer.java:525)
	at org.apache.solr.cloud.TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry(TestLeaderElectionZkExpiry.java:104)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.TestRandomRequestDistribution

Error Message:
org.apache.solr.cloud.TestRandomRequestDistribution

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.cloud.TestRandomRequestDistribution
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:273)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:233)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:355)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/cloud/TestRandomRequestDistribution.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.TestReplicaProperties

Error Message:
org.apache.solr.cloud.TestReplicaProperties

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.cloud.TestReplicaProperties
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:273)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:233)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:355)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/cloud/TestReplicaProperties.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.TestRequestStatusCollectionAPI

Error Message:
org.apache.solr.cloud.TestRequestStatusCollectionAPI

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.cloud.TestRequestStatusCollectionAPI
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:273)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:233)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:355)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/cloud/TestRequestStatusCollectionAPI.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.TestSSLRandomization

Error Message:
org.apache.solr.cloud.TestSSLRandomization

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.cloud.TestSSLRandomization
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:273)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:233)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:355)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/cloud/TestSSLRandomization.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.TestSizeLimitedDistributedMap

Error Message:
org.apache.solr.cloud.TestSizeLimitedDistributedMap

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.cloud.TestSizeLimitedDistributedMap
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:273)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:233)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:355)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/cloud/TestSizeLimitedDistributedMap.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates

Error Message:
org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:273)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:233)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:355)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/cloud/TestStressCloudBlindAtomicUpdates.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.TestStressLiveNodes

Error Message:
org.apache.solr.cloud.TestStressLiveNodes

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.cloud.TestStressLiveNodes
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:273)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:233)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:355)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/cloud/TestStressLiveNodes.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.TestTolerantUpdateProcessorRandomCloud

Error Message:
org.apache.solr.cloud.TestTolerantUpdateProcessorRandomCloud

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.cloud.TestTolerantUpdateProcessorRandomCloud
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:273)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:233)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:355)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/cloud/TestTolerantUpdateProcessorRandomCloud.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.TriLevelCompositeIdRoutingTest

Error Message:
org.apache.solr.cloud.TriLevelCompositeIdRoutingTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.cloud.TriLevelCompositeIdRoutingTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:273)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:233)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:355)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/cloud/TriLevelCompositeIdRoutingTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsBasicDistributedZk2Test

Error Message:
org.apache.solr.cloud.hdfs.HdfsBasicDistributedZk2Test

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.cloud.hdfs.HdfsBasicDistributedZk2Test
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:273)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:233)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:355)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/cloud/hdfs/HdfsBasicDistributedZk2Test.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsBasicDistributedZkTest

Error Message:
org.apache.solr.cloud.hdfs.HdfsBasicDistributedZkTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.cloud.hdfs.HdfsBasicDistributedZkTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:273)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:233)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:355)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/cloud/hdfs/HdfsBasicDistributedZkTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest

Error Message:
org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:273)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:233)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:355)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/cloud/hdfs/HdfsChaosMonkeySafeLeaderTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsNNFailoverTest

Error Message:
org.apache.solr.cloud.hdfs.HdfsNNFailoverTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.cloud.hdfs.HdfsNNFailoverTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:273)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:233)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:355)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/cloud/hdfs/HdfsNNFailoverTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsRecoverLeaseTest

Error Message:
org.apache.solr.cloud.hdfs.HdfsRecoverLeaseTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.cloud.hdfs.HdfsRecoverLeaseTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:273)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:233)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:355)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/cloud/hdfs/HdfsRecoverLeaseTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest

Error Message:
org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:273)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:233)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:355)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/cloud/hdfs/HdfsRecoveryZkTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsRestartWhileUpdatingTest

Error Message:
org.apache.solr.cloud.hdfs.HdfsRestartWhileUpdatingTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.cloud.hdfs.HdfsRestartWhileUpdatingTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:273)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:233)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:355)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/cloud/hdfs/HdfsRestartWhileUpdatingTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsTlogReplayBufferedWhileIndexingTest

Error Message:
org.apache.solr.cloud.hdfs.HdfsTlogReplayBufferedWhileIndexingTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.cloud.hdfs.HdfsTlogReplayBufferedWhileIndexingTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:273)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:233)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:355)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/cloud/hdfs/HdfsTlogReplayBufferedWhileIndexingTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsUnloadDistributedZkTest

Error Message:
org.apache.solr.cloud.hdfs.HdfsUnloadDistributedZkTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.cloud.hdfs.HdfsUnloadDistributedZkTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:273)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:233)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:355)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/cloud/hdfs/HdfsUnloadDistributedZkTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsWriteToMultipleCollectionsTest

Error Message:
org.apache.solr.cloud.hdfs.HdfsWriteToMultipleCollectionsTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.cloud.hdfs.HdfsWriteToMultipleCollectionsTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:273)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:233)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:355)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/cloud/hdfs/HdfsWriteToMultipleCollectionsTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.StressHdfsTest

Error Message:
org.apache.solr.cloud.hdfs.StressHdfsTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.cloud.hdfs.StressHdfsTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:273)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:233)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:355)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/cloud/hdfs/StressHdfsTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.overseer.ZkStateWriterTest

Error Message:
org.apache.solr.cloud.overseer.ZkStateWriterTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.cloud.overseer.ZkStateWriterTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:273)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:233)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:355)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/cloud/overseer/ZkStateWriterTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.rule.ImplicitSnitchTest

Error Message:
org.apache.solr.cloud.rule.ImplicitSnitchTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.cloud.rule.ImplicitSnitchTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:273)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:233)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:355)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/cloud/rule/ImplicitSnitchTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.core.BlobRepositoryCloudTest

Error Message:
org.apache.solr.core.BlobRepositoryCloudTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.core.BlobRepositoryCloudTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:273)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:233)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:355)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/core/BlobRepositoryCloudTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.core.ConfigureRecoveryStrategyTest

Error Message:
org.apache.solr.core.ConfigureRecoveryStrategyTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.core.ConfigureRecoveryStrategyTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:273)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:233)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:355)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/core/ConfigureRecoveryStrategyTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.core.HdfsDirectoryFactoryTest

Error Message:
org.apache.solr.core.HdfsDirectoryFactoryTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.core.HdfsDirectoryFactoryTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:273)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:233)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:355)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/core/HdfsDirectoryFactoryTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.core.QueryResultKeyTest

Error Message:
org.apache.solr.core.QueryResultKeyTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.core.QueryResultKeyTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:273)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:233)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:355)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/core/QueryResultKeyTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.core.TestBackupRepositoryFactory

Error Message:
org.apache.solr.core.TestBackupRepositoryFactory

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.core.TestBackupRepositoryFactory
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:273)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:233)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:355)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/core/TestBackupRepositoryFactory.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.core.TestDynamicLoading

Error Message:
org.apache.solr.core.TestDynamicLoading

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.core.TestDynamicLoading
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:273)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:233)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:355)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/core/TestDynamicLoading.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.core.TestInitParams

Error Message:
org.apache.solr.core.TestInitParams

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.core.TestInitParams
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:273)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:233)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:355)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/core/TestInitParams.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.core.TestNRTOpen

Error Message:
org.apache.solr.core.TestNRTOpen

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.core.TestNRTOpen
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:273)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:233)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:355)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/core/TestNRTOpen.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.core.TestReloadAndDeleteDocs

Error Message:
org.apache.solr.core.TestReloadAndDeleteDocs

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.core.TestReloadAndDeleteDocs
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:273)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:233)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:355)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/core/TestReloadAndDeleteDocs.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.core.TestSolrXml

Error Message:
org.apache.solr.core.TestSolrXml

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.core.TestSolrXml
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:273)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:233)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:355)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/core/TestSolrXml.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.handler.RequestLoggingTest

Error Message:
org.apache.solr.handler.RequestLoggingTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.handler.RequestLoggingTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:273)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:233)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:355)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/handler/RequestLoggingTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more




Build Log:
[...truncated 12593 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestDeleteCollectionOnDownNodes
   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestDeleteCollectionOnDownNodes_E1B1817A22BABE1A-001/init-core-data-001
   [junit4]   2> 2830975 WARN  (SUITE-TestDeleteCollectionOnDownNodes-seed#[E1B1817A22BABE1A]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=14 numCloses=14
   [junit4]   2> 2830975 INFO  (SUITE-TestDeleteCollectionOnDownNodes-seed#[E1B1817A22BABE1A]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields
   [junit4]   2> 2830980 INFO  (SUITE-TestDeleteCollectionOnDownNodes-seed#[E1B1817A22BABE1A]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth
   [junit4]   2> 2830982 INFO  (SUITE-TestDeleteCollectionOnDownNodes-seed#[E1B1817A22BABE1A]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 4 servers in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestDeleteCollectionOnDownNodes_E1B1817A22BABE1A-001/tempDir-001
   [junit4]   2> 2830982 INFO  (SUITE-TestDeleteCollectionOnDownNodes-seed#[E1B1817A22BABE1A]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 2830982 INFO  (Thread-4031) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 2830982 INFO  (Thread-4031) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 2831084 INFO  (SUITE-TestDeleteCollectionOnDownNodes-seed#[E1B1817A22BABE1A]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:57508
   [junit4]   2> 2831144 INFO  (jetty-launcher-4124-thread-4) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 2831145 INFO  (jetty-launcher-4124-thread-3) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 2831145 INFO  (jetty-launcher-4124-thread-1) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 2831145 INFO  (jetty-launcher-4124-thread-2) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 2831147 INFO  (jetty-launcher-4124-thread-4) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@131e1ee5{/solr,null,AVAILABLE}
   [junit4]   2> 2831148 INFO  (jetty-launcher-4124-thread-4) [    ] o.e.j.s.AbstractConnector Started ServerConnector@5f2e3b36{SSL,[ssl, http/1.1]}{127.0.0.1:57514}
   [junit4]   2> 2831148 INFO  (jetty-launcher-4124-thread-4) [    ] o.e.j.s.Server Started @2837774ms
   [junit4]   2> 2831148 INFO  (jetty-launcher-4124-thread-4) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=57514}
   [junit4]   2> 2831148 ERROR (jetty-launcher-4124-thread-4) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2831149 INFO  (jetty-launcher-4124-thread-4) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 6.6.0
   [junit4]   2> 2831149 INFO  (jetty-launcher-4124-thread-4) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2831149 INFO  (jetty-launcher-4124-thread-4) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2831149 INFO  (jetty-launcher-4124-thread-4) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-03-19T22:15:46.237Z
   [junit4]   2> 2831153 INFO  (jetty-launcher-4124-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3e2716a9{/solr,null,AVAILABLE}
   [junit4]   2> 2831154 INFO  (jetty-launcher-4124-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@447a751{SSL,[ssl, http/1.1]}{127.0.0.1:57516}
   [junit4]   2> 2831154 INFO  (jetty-launcher-4124-thread-1) [    ] o.e.j.s.Server Started @2837780ms
   [junit4]   2> 2831154 INFO  (jetty-launcher-4124-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=57516}
   [junit4]   2> 2831154 ERROR (jetty-launcher-4124-thread-1) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2831156 INFO  (jetty-launcher-4124-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 6.6.0
   [junit4]   2> 2831157 INFO  (jetty-launcher-4124-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2831157 INFO  (jetty-launcher-4124-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2831157 INFO  (jetty-launcher-4124-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-03-19T22:15:46.245Z
   [junit4]   2> 2831157 INFO  (jetty-launcher-4124-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7ae5ebbf{/solr,null,AVAILABLE}
   [junit4]   2> 2831157 INFO  (jetty-launcher-4124-thread-3) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4f6df90b{/solr,null,AVAILABLE}
   [junit4]   2> 2831157 INFO  (jetty-launcher-4124-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@754adc5a{SSL,[ssl, http/1.1]}{127.0.0.1:57517}
   [junit4]   2> 2831157 INFO  (jetty-launcher-4124-thread-2) [    ] o.e.j.s.Server Started @2837784ms
   [junit4]   2> 2831157 INFO  (jetty-launcher-4124-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=57517}
   [junit4]   2> 2831158 ERROR (jetty-launcher-4124-thread-2) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2831158 INFO  (jetty-launcher-4124-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 6.6.0
   [junit4]   2> 2831158 INFO  (jetty-launcher-4124-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2831158 INFO  (jetty-launcher-4124-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2831158 INFO  (jetty-launcher-4124-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-03-19T22:15:46.246Z
   [junit4]   2> 2831159 INFO  (jetty-launcher-4124-thread-3) [    ] o.e.j.s.AbstractConnector Started ServerConnector@6059febe{SSL,[ssl, http/1.1]}{127.0.0.1:57518}
   [junit4]   2> 2831159 INFO  (jetty-launcher-4124-thread-3) [    ] o.e.j.s.Server Started @2837785ms
   [junit4]   2> 2831159 INFO  (jetty-launcher-4124-thread-3) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=57518}
   [junit4]   2> 2831159 ERROR (jetty-launcher-4124-thread-3) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2831159 INFO  (jetty-launcher-4124-thread-3) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 6.6.0
   [junit4]   2> 2831159 INFO  (jetty-launcher-4124-thread-3) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2831159 INFO  (jetty-launcher-4124-thread-3) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2831160 INFO  (jetty-launcher-4124-thread-3) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-03-19T22:15:46.248Z
   [junit4]   2> 2831163 INFO  (jetty-launcher-4124-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2831164 INFO  (jetty-launcher-4124-thread-4) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2831164 INFO  (jetty-launcher-4124-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2831164 INFO  (jetty-launcher-4124-thread-3) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2831180 INFO  (jetty-launcher-4124-thread-4) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 2831181 INFO  (jetty-launcher-4124-thread-3) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 2831181 INFO  (jetty-launcher-4124-thread-2) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 2831181 INFO  (jetty-launcher-4124-thread-1) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 2831182 INFO  (jetty-launcher-4124-thread-3) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:57508/solr
   [junit4]   2> 2831182 INFO  (jetty-launcher-4124-thread-4) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:57508/solr
   [junit4]   2> 2831183 INFO  (jetty-launcher-4124-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:57508/solr
   [junit4]   2> 2831185 INFO  (jetty-launcher-4124-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:57508/solr
   [junit4]   2> 2831268 INFO  (jetty-launcher-4124-thread-3) [n:127.0.0.1:57518_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2831268 INFO  (jetty-launcher-4124-thread-2) [n:127.0.0.1:57517_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2831269 INFO  (jetty-launcher-4124-thread-3) [n:127.0.0.1:57518_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:57518_solr
   [junit4]   2> 2831270 INFO  (jetty-launcher-4124-thread-2) [n:127.0.0.1:57517_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57517_solr
   [junit4]   2> 2831271 INFO  (jetty-launcher-4124-thread-3) [n:127.0.0.1:57518_solr    ] o.a.s.c.Overseer Overseer (id=97646132990640137-127.0.0.1:57518_solr-n_0000000000) starting
   [junit4]   2> 2831275 INFO  (zkCallback-4143-thread-1-processing-n:127.0.0.1:57518_solr) [n:127.0.0.1:57518_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2831275 INFO  (zkCallback-4146-thread-1-processing-n:127.0.0.1:57517_solr) [n:127.0.0.1:57517_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2831277 INFO  (zkCallback-4145-thread-1-processing-n:127.0.0.1:57514_solr) [n:127.0.0.1:57514_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2831280 INFO  (zkCallback-4144-thread-1-processing-n:127.0.0.1:57516_solr) [n:127.0.0.1:57516_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2831283 INFO  (jetty-launcher-4124-thread-4) [n:127.0.0.1:57514_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2831283 INFO  (jetty-launcher-4124-thread-1) [n:127.0.0.1:57516_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2831297 INFO  (jetty-launcher-4124-thread-3) [n:127.0.0.1:57518_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57518_solr
   [junit4]   2> 2831300 INFO  (zkCallback-4146-thread-1-processing-n:127.0.0.1:57517_solr) [n:127.0.0.1:57517_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2831301 INFO  (zkCallback-4144-thread-1-processing-n:127.0.0.1:57516_solr) [n:127.0.0.1:57516_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2831301 INFO  (zkCallback-4145-thread-1-processing-n:127.0.0.1:57514_solr) [n:127.0.0.1:57514_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2831301 INFO  (zkCallback-4143-thread-1-processing-n:127.0.0.1:57518_solr) [n:127.0.0.1:57518_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2831322 INFO  (jetty-launcher-4124-thread-4) [n:127.0.0.1:57514_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57514_solr
   [junit4]   2> 2831326 INFO  (zkCallback-4146-thread-1-processing-n:127.0.0.1:57517_solr) [n:127.0.0.1:57517_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2831326 INFO  (zkCallback-4145-thread-1-processing-n:127.0.0.1:57514_solr) [n:127.0.0.1:57514_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2831327 INFO  (zkCallback-4144-thread-1-processing-n:127.0.0.1:57516_solr) [n:127.0.0.1:57516_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2831327 INFO  (zkCallback-4143-thread-2-processing-n:127.0.0.1:57518_solr) [n:127.0.0.1:57518_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2831338 INFO  (jetty-launcher-4124-thread-1) [n:127.0.0.1:57516_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57516_solr
   [junit4]   2> 2831341 INFO  (zkCallback-4144-thread-1-processing-n:127.0.0.1:57516_solr) [n:127.0.0.1:57516_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 2831341 INFO  (zkCallback-4146-thread-1-processing-n:127.0.0.1:57517_solr) [n:127.0.0.1:57517_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 2831341 INFO  (zkCallback-4145-thread-1-processing-n:127.0.0.1:57514_solr) [n:127.0.0.1:57514_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 2831342 INFO  (zkCallback-4143-thread-2-processing-n:127.0.0.1:57518_solr) [n:127.0.0.1:57518_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 2831465 INFO  (jetty-launcher-4124-thread-2) [n:127.0.0.1:57517_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestDeleteCollectionOnDownNodes_E1B1817A22BABE1A-001/tempDir-001/node2/.
   [junit4]   2> 2831592 INFO  (jetty-launcher-4124-thread-3) [n:127.0.0.1:57518_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestDeleteCollectionOnDownNodes_E1B1817A22BABE1A-001/tempDir-001/node3/.
   [junit4]   2> 2831596 INFO  (jetty-launcher-4124-thread-1) [n:127.0.0.1:57516_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestDeleteCollectionOnDownNodes_E1B1817A22BABE1A-001/tempDir-001/node1/.
   [junit4]   2> 2831665 INFO  (jetty-launcher-4124-thread-4) [n:127.0.0.1:57514_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestDeleteCollectionOnDownNodes_E1B1817A22BABE1A-001/tempDir-001/node4/.
   [junit4]   2> 2834445 INFO  (SUITE-TestDeleteCollectionOnDownNodes-seed#[E1B1817A22BABE1A]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 2834447 INFO  (SUITE-TestDeleteCollectionOnDownNodes-seed#[E1B1817A22BABE1A]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:57508/solr ready
   [junit4]   2> 2834487 INFO  (TEST-TestDeleteCollectionOnDownNodes.deleteCollectionWithDownNodes-seed#[E1B1817A22BABE1A]) [    ] o.a.s.SolrTestCaseJ4 ###Starting deleteCollectionWithDownNodes
   [junit4]   2> 2834513 INFO  (qtp167521529-19752) [n:127.0.0.1:57514_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=3&collection.configName=conf&maxShardsPerNode=3&name=halfdeletedcollection2&action=CREATE&numShards=4&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2834516 INFO  (OverseerThreadFactory-9184-thread-1-processing-n:127.0.0.1:57518_solr) [n:127.0.0.1:57518_solr    ] o.a.s.c.CreateCollectionCmd Create collection halfdeletedcollection2
   [junit4]   2> 2834663 INFO  (qtp167521529-19753) [n:127.0.0.1:57514_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=halfdeletedcollection2_shard1_replica1&action=CREATE&numShards=4&collection=halfdeletedcollection2&shard=shard1&wt=javabin&version=2
   [junit4]   2> 2834735 INFO  (qtp284845408-19767) [n:127.0.0.1:57518_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=halfdeletedcollection2_shard2_replica3&action=CREATE&numShards=4&collection=halfdeletedcollection2&shard=shard2&wt=javabin&version=2
   [junit4]   2> 2834764 INFO  (qtp167521529-19750) [n:127.0.0.1:57514_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=halfdeletedcollection2_shard2_replica2&action=CREATE&numShards=4&collection=halfdeletedcollection2&shard=shard2&wt=javabin&version=2
   [junit4]   2> 2834792 INFO  (qtp167521529-19751) [n:127.0.0.1:57514_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=halfdeletedcollection2_shard3_replica3&action=CREATE&numShards=4&collection=halfdeletedcollection2&shard=shard3&wt=javabin&version=2
   [junit4]   2> 2834808 INFO  (qtp1082303594-19762) [n:127.0.0.1:57516_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=halfdeletedcollection2_shard4_replica3&action=CREATE&numShards=4&collection=halfdeletedcollection2&shard=shard4&wt=javabin&version=2
   [junit4]   2> 2834810 INFO  (qtp694585632-19777) [n:127.0.0.1:57517_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=halfdeletedcollection2_shard1_replica3&action=CREATE&numShards=4&collection=halfdeletedcollection2&shard=shard1&wt=javabin&version=2
   [junit4]   2> 2834810 INFO  (qtp1082303594-19774) [n:127.0.0.1:57516_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=halfdeletedcollection2_shard2_replica1&action=CREATE&numShards=4&collection=halfdeletedcollection2&shard=shard2&wt=javabin&version=2
   [junit4]   2> 2834820 INFO  (qtp1082303594-19773) [n:127.0.0.1:57516_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=halfdeletedcollection2_shard3_replica2&action=CREATE&numShards=4&collection=halfdeletedcollection2&shard=shard3&wt=javabin&version=2
   [junit4]   2> 2834839 INFO  (qtp284845408-19772) [n:127.0.0.1:57518_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=halfdeletedcollection2_shard4_replica1&action=CREATE&numShards=4&collection=halfdeletedcollection2&shard=shard4&wt=javabin&version=2
   [junit4]   2> 2834839 INFO  (qtp694585632-19780) [n:127.0.0.1:57517_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=halfdeletedcollection2_shard4_replica2&action=CREATE&numShards=4&collection=halfdeletedcollection2&shard=shard4&wt=javabin&version=2
   [junit4]   2> 2834840 INFO  (qtp694585632-19781) [n:127.0.0.1:57517_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=halfdelete

[...truncated too long message...]

lassLoader.java:454)
   [junit4]    > 	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
   [junit4]    > 	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
   [junit4]    > 	... 12 more
   [junit4] Completed [560/700 (30!)] on J1 in 0.00s, 0 tests, 1 error <<< FAILURES!

[...truncated 1 lines...]
   [junit4] Suite: org.apache.solr.core.TestInitParams
   [junit4] ERROR   0.00s J1 | TestInitParams (suite) <<<
   [junit4]    > Throwable #1: java.lang.ClassNotFoundException: org.apache.solr.core.TestInitParams
   [junit4]    > 	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
   [junit4]    > 	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
   [junit4]    > 	at java.security.AccessController.doPrivileged(Native Method)
   [junit4]    > 	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
   [junit4]    > 	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
   [junit4]    > 	at java.lang.Class.forName0(Native Method)
   [junit4]    > 	at java.lang.Class.forName(Class.java:348)
   [junit4]    > Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/core/TestInitParams.class (Too many open files)
   [junit4]    > 	at java.io.FileInputStream.open0(Native Method)
   [junit4]    > 	at java.io.FileInputStream.open(FileInputStream.java:195)
   [junit4]    > 	at java.io.FileInputStream.<init>(FileInputStream.java:138)
   [junit4]    > 	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
   [junit4]    > 	at sun.misc.Resource.cachedInputStream(Resource.java:77)
   [junit4]    > 	at sun.misc.Resource.getByteBuffer(Resource.java:160)
   [junit4]    > 	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
   [junit4]    > 	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
   [junit4]    > 	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
   [junit4]    > 	... 12 more
   [junit4] Completed [561/700 (31!)] on J1 in 0.00s, 0 tests, 1 error <<< FAILURES!

[...truncated 1 lines...]
   [junit4] Suite: org.apache.solr.core.TestNRTOpen
   [junit4] ERROR   0.00s J1 | TestNRTOpen (suite) <<<
   [junit4]    > Throwable #1: java.lang.ClassNotFoundException: org.apache.solr.core.TestNRTOpen
   [junit4]    > 	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
   [junit4]    > 	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
   [junit4]    > 	at java.security.AccessController.doPrivileged(Native Method)
   [junit4]    > 	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
   [junit4]    > 	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
   [junit4]    > 	at java.lang.Class.forName0(Native Method)
   [junit4]    > 	at java.lang.Class.forName(Class.java:348)
   [junit4]    > Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/core/TestNRTOpen.class (Too many open files)
   [junit4]    > 	at java.io.FileInputStream.open0(Native Method)
   [junit4]    > 	at java.io.FileInputStream.open(FileInputStream.java:195)
   [junit4]    > 	at java.io.FileInputStream.<init>(FileInputStream.java:138)
   [junit4]    > 	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
   [junit4]    > 	at sun.misc.Resource.cachedInputStream(Resource.java:77)
   [junit4]    > 	at sun.misc.Resource.getByteBuffer(Resource.java:160)
   [junit4]    > 	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
   [junit4]    > 	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
   [junit4]    > 	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
   [junit4]    > 	... 12 more
   [junit4] Completed [562/700 (32!)] on J1 in 0.00s, 0 tests, 1 error <<< FAILURES!

[...truncated 1 lines...]
   [junit4] Suite: org.apache.solr.core.TestReloadAndDeleteDocs
   [junit4] ERROR   0.00s J1 | TestReloadAndDeleteDocs (suite) <<<
   [junit4]    > Throwable #1: java.lang.ClassNotFoundException: org.apache.solr.core.TestReloadAndDeleteDocs
   [junit4]    > 	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
   [junit4]    > 	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
   [junit4]    > 	at java.security.AccessController.doPrivileged(Native Method)
   [junit4]    > 	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
   [junit4]    > 	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
   [junit4]    > 	at java.lang.Class.forName0(Native Method)
   [junit4]    > 	at java.lang.Class.forName(Class.java:348)
   [junit4]    > Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/core/TestReloadAndDeleteDocs.class (Too many open files)
   [junit4]    > 	at java.io.FileInputStream.open0(Native Method)
   [junit4]    > 	at java.io.FileInputStream.open(FileInputStream.java:195)
   [junit4]    > 	at java.io.FileInputStream.<init>(FileInputStream.java:138)
   [junit4]    > 	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
   [junit4]    > 	at sun.misc.Resource.cachedInputStream(Resource.java:77)
   [junit4]    > 	at sun.misc.Resource.getByteBuffer(Resource.java:160)
   [junit4]    > 	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
   [junit4]    > 	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
   [junit4]    > 	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
   [junit4]    > 	... 12 more
   [junit4] Completed [563/700 (33!)] on J1 in 0.00s, 0 tests, 1 error <<< FAILURES!

[...truncated 1 lines...]
   [junit4] Suite: org.apache.solr.core.TestSolrXml
   [junit4] ERROR   0.00s J1 | TestSolrXml (suite) <<<
   [junit4]    > Throwable #1: java.lang.ClassNotFoundException: org.apache.solr.core.TestSolrXml
   [junit4]    > 	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
   [junit4]    > 	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
   [junit4]    > 	at java.security.AccessController.doPrivileged(Native Method)
   [junit4]    > 	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
   [junit4]    > 	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
   [junit4]    > 	at java.lang.Class.forName0(Native Method)
   [junit4]    > 	at java.lang.Class.forName(Class.java:348)
   [junit4]    > Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/core/TestSolrXml.class (Too many open files)
   [junit4]    > 	at java.io.FileInputStream.open0(Native Method)
   [junit4]    > 	at java.io.FileInputStream.open(FileInputStream.java:195)
   [junit4]    > 	at java.io.FileInputStream.<init>(FileInputStream.java:138)
   [junit4]    > 	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
   [junit4]    > 	at sun.misc.Resource.cachedInputStream(Resource.java:77)
   [junit4]    > 	at sun.misc.Resource.getByteBuffer(Resource.java:160)
   [junit4]    > 	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
   [junit4]    > 	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
   [junit4]    > 	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
   [junit4]    > 	... 12 more
   [junit4] Completed [564/700 (34!)] on J1 in 0.00s, 0 tests, 1 error <<< FAILURES!

[...truncated 1 lines...]
   [junit4] Suite: org.apache.solr.handler.RequestLoggingTest
   [junit4] ERROR   0.00s J1 | RequestLoggingTest (suite) <<<
   [junit4]    > Throwable #1: java.lang.ClassNotFoundException: org.apache.solr.handler.RequestLoggingTest
   [junit4]    > 	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
   [junit4]    > 	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
   [junit4]    > 	at java.security.AccessController.doPrivileged(Native Method)
   [junit4]    > 	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
   [junit4]    > 	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
   [junit4]    > 	at java.lang.Class.forName0(Native Method)
   [junit4]    > 	at java.lang.Class.forName(Class.java:348)
   [junit4]    > Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/handler/RequestLoggingTest.class (Too many open files)
   [junit4]    > 	at java.io.FileInputStream.open0(Native Method)
   [junit4]    > 	at java.io.FileInputStream.open(FileInputStream.java:195)
   [junit4]    > 	at java.io.FileInputStream.<init>(FileInputStream.java:138)
   [junit4]    > 	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
   [junit4]    > 	at sun.misc.Resource.cachedInputStream(Resource.java:77)
   [junit4]    > 	at sun.misc.Resource.getByteBuffer(Resource.java:160)
   [junit4]    > 	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
   [junit4]    > 	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
   [junit4]    > 	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
   [junit4]    > 	... 12 more
   [junit4] Completed [565/700 (35!)] on J1 in 0.00s, 0 tests, 1 error <<< FAILURES!

[...truncated 63161 lines...]