You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2017/08/18 13:12:24 UTC

[JENKINS] Lucene-Solr-7.x-MacOSX (64bit/jdk1.8.0) - Build # 119 - Still Unstable!

Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-MacOSX/119/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseSerialGC

3 tests failed.
FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
Doc with id=1 not found in http://127.0.0.1:60002/collMinRf_1x3 due to: Path not found: /id; rsp={doc=null}

Stack Trace:
java.lang.AssertionError: Doc with id=1 not found in http://127.0.0.1:60002/collMinRf_1x3 due to: Path not found: /id; rsp={doc=null}
	at __randomizedtesting.SeedInfo.seed([D6A2CC4B3E46D91F:5EF6F39190BAB4E7]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.cloud.HttpPartitionTest.assertDocExists(HttpPartitionTest.java:603)
	at org.apache.solr.cloud.HttpPartitionTest.assertDocsExistInAllReplicas(HttpPartitionTest.java:558)
	at org.apache.solr.cloud.HttpPartitionTest.testMinRf(HttpPartitionTest.java:249)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:127)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.TestTlogReplica.testOutOfOrderDBQWithInPlaceUpdates

Error Message:


Stack Trace:
java.lang.NullPointerException
	at __randomizedtesting.SeedInfo.seed([D6A2CC4B3E46D91F:506334A661170FFF]:0)
	at org.apache.solr.cloud.TestTlogReplica.getSolrCore(TestTlogReplica.java:846)
	at org.apache.solr.cloud.TestTlogReplica.testOutOfOrderDBQWithInPlaceUpdates(TestTlogReplica.java:652)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.TestTlogReplica.testOnlyLeaderIndexes

Error Message:


Stack Trace:
java.lang.NullPointerException
	at __randomizedtesting.SeedInfo.seed([D6A2CC4B3E46D91F:CAA3B1C64BE3A78C]:0)
	at org.apache.solr.cloud.TestTlogReplica.getSolrCore(TestTlogReplica.java:846)
	at org.apache.solr.cloud.TestTlogReplica.testOnlyLeaderIndexes(TestTlogReplica.java:467)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 11359 lines...]
   [junit4] Suite: org.apache.solr.cloud.HttpPartitionTest
   [junit4]   2> 774573 INFO  (SUITE-HttpPartitionTest-seed#[D6A2CC4B3E46D91F]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_D6A2CC4B3E46D91F-001/init-core-data-001
   [junit4]   2> 774574 WARN  (SUITE-HttpPartitionTest-seed#[D6A2CC4B3E46D91F]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
   [junit4]   2> 774574 INFO  (SUITE-HttpPartitionTest-seed#[D6A2CC4B3E46D91F]-worker) [    ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 774577 INFO  (SUITE-HttpPartitionTest-seed#[D6A2CC4B3E46D91F]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776) w/ MAC_OS_X supressed clientAuth
   [junit4]   2> 774577 INFO  (SUITE-HttpPartitionTest-seed#[D6A2CC4B3E46D91F]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
   [junit4]   2> 774580 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 774580 INFO  (Thread-1408) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 774580 INFO  (Thread-1408) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 774582 ERROR (Thread-1408) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 774682 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.c.ZkTestServer start zk server on port:59960
   [junit4]   2> 774731 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 774733 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 774736 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 774738 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 774740 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 774743 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 774745 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 774748 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 774750 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 774752 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 774754 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 774756 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Will use TLOG replicas unless explicitly asked otherwise
   [junit4]   2> 775066 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 775067 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@113014fd{/,null,AVAILABLE}
   [junit4]   2> 775072 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@308ecab7{HTTP/1.1,[http/1.1]}{127.0.0.1:59966}
   [junit4]   2> 775074 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.e.j.s.Server Started @779671ms
   [junit4]   2> 775074 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_D6A2CC4B3E46D91F-001/tempDir-001/control/data, replicaType=NRT, hostContext=/, hostPort=59965, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_D6A2CC4B3E46D91F-001/control-001/cores}
   [junit4]   2> 775074 ERROR (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 775074 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.1.0
   [junit4]   2> 775074 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 775074 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 775074 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-08-18T12:10:01.661Z
   [junit4]   2> 775078 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 775078 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_D6A2CC4B3E46D91F-001/control-001/solr.xml
   [junit4]   2> 775086 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@235c5dfc, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 775092 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:59960/solr
   [junit4]   2> 775236 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [n:127.0.0.1:59965_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 775237 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [n:127.0.0.1:59965_    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:59965_
   [junit4]   2> 775239 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [n:127.0.0.1:59965_    ] o.a.s.c.Overseer Overseer (id=98504422271746052-127.0.0.1:59965_-n_0000000000) starting
   [junit4]   2> 775251 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [n:127.0.0.1:59965_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:59965_
   [junit4]   2> 775256 INFO  (zkCallback-1059-thread-1-processing-n:127.0.0.1:59965_) [n:127.0.0.1:59965_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 775336 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [n:127.0.0.1:59965_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@235c5dfc
   [junit4]   2> 775346 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [n:127.0.0.1:59965_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@235c5dfc
   [junit4]   2> 775347 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [n:127.0.0.1:59965_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@235c5dfc
   [junit4]   2> 775349 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [n:127.0.0.1:59965_    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_D6A2CC4B3E46D91F-001/control-001/cores
   [junit4]   2> 775377 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 775378 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:59960/solr ready
   [junit4]   2> 775379 INFO  (SocketProxy-Acceptor-59965) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=59971,localport=59965], receiveBufferSize:408300
   [junit4]   2> 775388 INFO  (SocketProxy-Acceptor-59965) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=59966,localport=59972], receiveBufferSize=408300
   [junit4]   2> 775393 INFO  (qtp1718562341-6730) [n:127.0.0.1:59965_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:59965_&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 775396 INFO  (OverseerThreadFactory-2783-thread-1-processing-n:127.0.0.1:59965_) [n:127.0.0.1:59965_    ] o.a.s.c.CreateCollectionCmd Create collection control_collection
   [junit4]   2> 775504 INFO  (SocketProxy-Acceptor-59965) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=59973,localport=59965], receiveBufferSize:408300
   [junit4]   2> 775505 INFO  (SocketProxy-Acceptor-59965) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=59966,localport=59974], receiveBufferSize=408300
   [junit4]   2> 775508 INFO  (qtp1718562341-6732) [n:127.0.0.1:59965_    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 775509 INFO  (qtp1718562341-6732) [n:127.0.0.1:59965_    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 775618 INFO  (zkCallback-1059-thread-1-processing-n:127.0.0.1:59965_) [n:127.0.0.1:59965_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 776542 INFO  (qtp1718562341-6732) [n:127.0.0.1:59965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 776581 INFO  (qtp1718562341-6732) [n:127.0.0.1:59965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
   [junit4]   2> 776725 INFO  (qtp1718562341-6732) [n:127.0.0.1:59965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 776755 INFO  (qtp1718562341-6732) [n:127.0.0.1:59965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
   [junit4]   2> 776755 INFO  (qtp1718562341-6732) [n:127.0.0.1:59965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@235c5dfc
   [junit4]   2> 776755 INFO  (qtp1718562341-6732) [n:127.0.0.1:59965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 776755 INFO  (qtp1718562341-6732) [n:127.0.0.1:59965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_D6A2CC4B3E46D91F-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_D6A2CC4B3E46D91F-001/control-001/cores/control_collection_shard1_replica_n1/data/]
   [junit4]   2> 776762 INFO  (qtp1718562341-6732) [n:127.0.0.1:59965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1420090314, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 776774 WARN  (qtp1718562341-6732) [n:127.0.0.1:59965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 776862 INFO  (qtp1718562341-6732) [n:127.0.0.1:59965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 776862 INFO  (qtp1718562341-6732) [n:127.0.0.1:59965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 776864 INFO  (qtp1718562341-6732) [n:127.0.0.1:59965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 776864 INFO  (qtp1718562341-6732) [n:127.0.0.1:59965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 776866 INFO  (qtp1718562341-6732) [n:127.0.0.1:59965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=46, maxMergeAtOnceExplicit=23, maxMergedSegmentMB=27.3935546875, floorSegmentMB=1.404296875, forceMergeDeletesPctAllowed=20.937546942279567, segmentsPerTier=46.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6959952585937709
   [junit4]   2> 776868 INFO  (qtp1718562341-6732) [n:127.0.0.1:59965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6bcf508b[control_collection_shard1_replica_n1] main]
   [junit4]   2> 776870 INFO  (qtp1718562341-6732) [n:127.0.0.1:59965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 776871 INFO  (qtp1718562341-6732) [n:127.0.0.1:59965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 776872 INFO  (qtp1718562341-6732) [n:127.0.0.1:59965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 776872 INFO  (qtp1718562341-6732) [n:127.0.0.1:59965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1576070758750224384
   [junit4]   2> 776874 INFO  (searcherExecutor-2786-thread-1-processing-n:127.0.0.1:59965_ x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:59965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@6bcf508b[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 776888 INFO  (qtp1718562341-6732) [n:127.0.0.1:59965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 776888 INFO  (qtp1718562341-6732) [n:127.0.0.1:59965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 776888 INFO  (qtp1718562341-6732) [n:127.0.0.1:59965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:59965/control_collection_shard1_replica_n1/
   [junit4]   2> 776888 INFO  (qtp1718562341-6732) [n:127.0.0.1:59965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 776888 INFO  (qtp1718562341-6732) [n:127.0.0.1:59965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:59965/control_collection_shard1_replica_n1/ has no replicas
   [junit4]   2> 776888 INFO  (qtp1718562341-6732) [n:127.0.0.1:59965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 776896 INFO  (qtp1718562341-6732) [n:127.0.0.1:59965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:59965/control_collection_shard1_replica_n1/ shard1
   [junit4]   2> 777002 INFO  (zkCallback-1059-thread-1-processing-n:127.0.0.1:59965_) [n:127.0.0.1:59965_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 777054 INFO  (qtp1718562341-6732) [n:127.0.0.1:59965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 777056 INFO  (qtp1718562341-6732) [n:127.0.0.1:59965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1547
   [junit4]   2> 777063 INFO  (qtp1718562341-6730) [n:127.0.0.1:59965_    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 777172 INFO  (zkCallback-1059-thread-1-processing-n:127.0.0.1:59965_) [n:127.0.0.1:59965_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 777402 INFO  (OverseerCollectionConfigSetProcessor-98504422271746052-127.0.0.1:59965_-n_0000000000) [n:127.0.0.1:59965_    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 778073 INFO  (qtp1718562341-6730) [n:127.0.0.1:59965_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:59965_&wt=javabin&version=2} status=0 QTime=2680
   [junit4]   2> 778082 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 778083 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:59960/solr ready
   [junit4]   2> 778083 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 778084 INFO  (SocketProxy-Acceptor-59965) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=59977,localport=59965], receiveBufferSize:408300
   [junit4]   2> 778085 INFO  (SocketProxy-Acceptor-59965) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=59966,localport=59978], receiveBufferSize=408300
   [junit4]   2> 778088 INFO  (qtp1718562341-6735) [n:127.0.0.1:59965_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 778091 INFO  (OverseerThreadFactory-2783-thread-2-processing-n:127.0.0.1:59965_) [n:127.0.0.1:59965_    ] o.a.s.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 778092 WARN  (OverseerThreadFactory-2783-thread-2-processing-n:127.0.0.1:59965_) [n:127.0.0.1:59965_    ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
   [junit4]   2> 778300 INFO  (qtp1718562341-6735) [n:127.0.0.1:59965_    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 778300 INFO  (qtp1718562341-6735) [n:127.0.0.1:59965_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=212
   [junit4]   2> 778599 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_D6A2CC4B3E46D91F-001/shard-1-001 of type TLOG
   [junit4]   2> 778599 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 778601 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@36b22df0{/,null,AVAILABLE}
   [junit4]   2> 778601 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@5c9f47e2{HTTP/1.1,[http/1.1]}{127.0.0.1:59980}
   [junit4]   2> 778601 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.e.j.s.Server Started @783199ms
   [junit4]   2> 778601 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_D6A2CC4B3E46D91F-001/tempDir-001/jetty1, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/, hostPort=59979, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_D6A2CC4B3E46D91F-001/shard-1-001/cores}
   [junit4]   2> 778603 ERROR (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 778603 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.1.0
   [junit4]   2> 778603 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 778603 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 778603 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-08-18T12:10:05.190Z
   [junit4]   2> 778608 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 778608 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_D6A2CC4B3E46D91F-001/shard-1-001/solr.xml
   [junit4]   2> 778616 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@235c5dfc, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 778620 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:59960/solr
   [junit4]   2> 778633 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [n:127.0.0.1:59979_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 778637 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [n:127.0.0.1:59979_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 778639 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [n:127.0.0.1:59979_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:59979_
   [junit4]   2> 778641 INFO  (zkCallback-1059-thread-1-processing-n:127.0.0.1:59965_) [n:127.0.0.1:59965_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 778643 INFO  (zkCallback-1071-thread-1-processing-n:127.0.0.1:59979_) [n:127.0.0.1:59979_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 778641 INFO  (zkCallback-1066-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 778715 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [n:127.0.0.1:59979_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@235c5dfc
   [junit4]   2> 778723 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [n:127.0.0.1:59979_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@235c5dfc
   [junit4]   2> 778723 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [n:127.0.0.1:59979_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@235c5dfc
   [junit4]   2> 778725 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [n:127.0.0.1:59979_    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_D6A2CC4B3E46D91F-001/shard-1-001/cores
   [junit4]   2> 778753 INFO  (SocketProxy-Acceptor-59979) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=59984,localport=59979], receiveBufferSize:408300
   [junit4]   2> 778755 INFO  (SocketProxy-Acceptor-59979) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=59980,localport=59985], receiveBufferSize=408300
   [junit4]   2> 778760 INFO  (qtp1632235631-6778) [n:127.0.0.1:59979_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:59979_&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 778763 INFO  (OverseerThreadFactory-2783-thread-3-processing-n:127.0.0.1:59965_) [n:127.0.0.1:59965_    ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:59979_ for creating new replica
   [junit4]   2> 778763 INFO  (OverseerCollectionConfigSetProcessor-98504422271746052-127.0.0.1:59965_-n_0000000000) [n:127.0.0.1:59965_    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 778766 INFO  (SocketProxy-Acceptor-59979) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=59986,localport=59979], receiveBufferSize:408300
   [junit4]   2> 778766 INFO  (SocketProxy-Acceptor-59979) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=59980,localport=59987], receiveBufferSize=408300
   [junit4]   2> 778770 INFO  (qtp1632235631-6780) [n:127.0.0.1:59979_    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 778771 INFO  (qtp1632235631-6780) [n:127.0.0.1:59979_    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 779818 INFO  (qtp1632235631-6780) [n:127.0.0.1:59979_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 779829 INFO  (qtp1632235631-6780) [n:127.0.0.1:59979_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.s.IndexSchema [collection1_shard2_replica_t41] Schema name=test
   [junit4]   2> 779948 INFO  (qtp1632235631-6780) [n:127.0.0.1:59979_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 779980 INFO  (qtp1632235631-6780) [n:127.0.0.1:59979_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_t41' using configuration from collection collection1, trusted=true
   [junit4]   2> 779980 INFO  (qtp1632235631-6780) [n:127.0.0.1:59979_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_t41' (registry 'solr.core.collection1.shard2.replica_t41') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@235c5dfc
   [junit4]   2> 779980 INFO  (qtp1632235631-6780) [n:127.0.0.1:59979_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 779980 INFO  (qtp1632235631-6780) [n:127.0.0.1:59979_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.SolrCore [[collection1_shard2_replica_t41] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_D6A2CC4B3E46D91F-001/shard-1-001/cores/collection1_shard2_replica_t41], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_D6A2CC4B3E46D91F-001/shard-1-001/cores/collection1_shard2_replica_t41/data/]
   [junit4]   2> 779983 INFO  (qtp1632235631-6780) [n:127.0.0.1:59979_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1420090314, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 779985 WARN  (qtp1632235631-6780) [n:127.0.0.1:59979_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 780070 INFO  (qtp1632235631-6780) [n:127.0.0.1:59979_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 780070 INFO  (qtp1632235631-6780) [n:127.0.0.1:59979_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 780071 INFO  (qtp1632235631-6780) [n:127.0.0.1:59979_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 780071 INFO  (qtp1632235631-6780) [n:127.0.0.1:59979_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 780073 INFO  (qtp1632235631-6780) [n:127.0.0.1:59979_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=46, maxMergeAtOnceExplicit=23, maxMergedSegmentMB=27.3935546875, floorSegmentMB=1.404296875, forceMergeDeletesPctAllowed=20.937546942279567, segmentsPerTier=46.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6959952585937709
   [junit4]   2> 780073 INFO  (qtp1632235631-6780) [n:127.0.0.1:59979_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.s.SolrIndexSearcher Opening [Searcher@628f046b[collection1_shard2_replica_t41] main]
   [junit4]   2> 780075 INFO  (qtp1632235631-6780) [n:127.0.0.1:59979_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 780075 INFO  (qtp1632235631-6780) [n:127.0.0.1:59979_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 780076 INFO  (qtp1632235631-6780) [n:127.0.0.1:59979_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 780076 INFO  (qtp1632235631-6780) [n:127.0.0.1:59979_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1576070762109861888
   [junit4]   2> 780079 INFO  (searcherExecutor-2797-thread-1-processing-n:127.0.0.1:59979_ x:collection1_shard2_replica_t41 s:shard2 c:collection1) [n:127.0.0.1:59979_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.SolrCore [collection1_shard2_replica_t41] Registered new searcher Searcher@628f046b[collection1_shard2_replica_t41] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 780088 INFO  (qtp1632235631-6780) [n:127.0.0.1:59979_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 780088 INFO  (qtp1632235631-6780) [n:127.0.0.1:59979_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 780088 INFO  (qtp1632235631-6780) [n:127.0.0.1:59979_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:59979/collection1_shard2_replica_t41/
   [junit4]   2> 780088 INFO  (qtp1632235631-6780) [n:127.0.0.1:59979_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 780088 INFO  (qtp1632235631-6780) [n:127.0.0.1:59979_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.SyncStrategy http://127.0.0.1:59979/collection1_shard2_replica_t41/ has no replicas
   [junit4]   2> 780088 INFO  (qtp1632235631-6780) [n:127.0.0.1:59979_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 780092 INFO  (qtp1632235631-6780) [n:127.0.0.1:59979_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.ZkController collection1_shard2_replica_t41 stopping background replication from leader
   [junit4]   2> 780098 INFO  (qtp1632235631-6780) [n:127.0.0.1:59979_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:59979/collection1_shard2_replica_t41/ shard2
   [junit4]   2> 780270 INFO  (qtp1632235631-6780) [n:127.0.0.1:59979_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 780272 INFO  (qtp1632235631-6780) [n:127.0.0.1:59979_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=1502
   [junit4]   2> 780281 INFO  (qtp1632235631-6778) [n:127.0.0.1:59979_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:59979_&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2} status=0 QTime=1521
   [junit4]   2> 780774 INFO  (OverseerCollectionConfigSetProcessor-98504422271746052-127.0.0.1:59965_-n_0000000000) [n:127.0.0.1:59965_    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 780776 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_D6A2CC4B3E46D91F-001/shard-2-001 of type TLOG
   [junit4]   2> 780776 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 780778 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@18db8814{/,null,AVAILABLE}
   [junit4]   2> 780779 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@8f07119{HTTP/1.1,[http/1.1]}{127.0.0.1:59991}
   [junit4]   2> 780779 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.e.j.s.Server Started @785377ms
   [junit4]   2> 780781 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_D6A2CC4B3E46D91F-001/tempDir-001/jetty2, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/, hostPort=59990, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_D6A2CC4B3E46D91F-001/shard-2-001/cores}
   [junit4]   2> 780782 ERROR (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 780785 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.1.0
   [junit4]   2> 780785 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 780785 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 780785 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-08-18T12:10:07.372Z
   [junit4]   2> 780797 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 780797 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_D6A2CC4B3E46D91F-001/shard-2-001/solr.xml
   [junit4]   2> 780805 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@235c5dfc, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 780811 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:59960/solr
   [junit4]   2> 780835 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [n:127.0.0.1:59990_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 780841 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [n:127.0.0.1:59990_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 780846 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [n:127.0.0.1:59990_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:59990_
   [junit4]   2> 780849 INFO  (zkCallback-1066-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 780849 INFO  (zkCallback-1059-thread-1-processing-n:127.0.0.1:59965_) [n:127.0.0.1:59965_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 780849 INFO  (zkCallback-1071-thread-1-processing-n:127.0.0.1:59979_) [n:127.0.0.1:59979_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 780854 INFO  (zkCallback-1077-thread-1-processing-n:127.0.0.1:59990_) [n:127.0.0.1:59990_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 781003 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [n:127.0.0.1:59990_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@235c5dfc
   [junit4]   2> 781034 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [n:127.0.0.1:59990_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@235c5dfc
   [junit4]   2> 781034 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [n:127.0.0.1:59990_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@235c5dfc
   [junit4]   2> 781037 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [n:127.0.0.1:59990_    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_D6A2CC4B3E46D91F-001/shard-2-001/cores
   [junit4]   2> 781084 INFO  (SocketProxy-Acceptor-59990) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=59995,localport=59990], receiveBufferSize:408300
   [junit4]   2> 781085 INFO  (SocketProxy-Acceptor-59990) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=59991,localport=59996], receiveBufferSize=408300
   [junit4]   2> 781092 INFO  (qtp476951373-6812) [n:127.0.0.1:59990_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:59990_&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 781096 INFO  (OverseerThreadFactory-2783-thread-4-processing-n:127.0.0.1:59965_) [n:127.0.0.1:59965_    ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:59990_ for creating new replica
   [junit4]   2> 781099 INFO  (SocketProxy-Acceptor-59990) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=59997,localport=59990], receiveBufferSize:408300
   [junit4]   2> 781100 INFO  (SocketProxy-Acceptor-59990) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=59991,localport=59998], receiveBufferSize=408300
   [junit4]   2> 781109 INFO  (qtp476951373-6814) [n:127.0.0.1:59990_    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t43&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 781109 INFO  (qtp476951373-6814) [n:127.0.0.1:59990_    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 782137 INFO  (qtp476951373-6814) [n:127.0.0.1:59990_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 782150 INFO  (qtp476951373-6814) [n:127.0.0.1:59990_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.s.IndexSchema [collection1_shard1_replica_t43] Schema name=test
   [junit4]   2> 782311 INFO  (qtp476951373-6814) [n:127.0.0.1:59990_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 782364 INFO  (qtp476951373-6814) [n:127.0.0.1:59990_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_t43' using configuration from collection collection1, trusted=true
   [junit4]   2> 782365 INFO  (qtp476951373-6814) [n:127.0.0.1:59990_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_t43' (registry 'solr.core.collection1.shard1.replica_t43') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@235c5dfc
   [junit4]   2> 782365 INFO  (qtp476951373-6814) [n:127.0.0.1:59990_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 782365 INFO  (qtp476951373-6814) [n:127.0.0.1:59990_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.SolrCore [[collection1_shard1_replica_t43] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_D6A2CC4B3E46D91F-001/shard-2-001/cores/collection1_shard1_replica_t43], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_D6A2CC4B3E46D91F-001/shard-2-001/cores/collection1_shard1_replica_t43/data/]
   [junit4]   2> 782369 INFO  (qtp476951373-6814) [n:127.0.0.1:59990_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1420090314, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 782375 WARN  (qtp476951373-6814) [n:127.0.0.1:59990_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 782461 INFO  (qtp476951373-6814) [n:127.0.0.1:59990_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 782461 INFO  (qtp476951373-6814) [n:127.0.0.1:59990_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 782462 INFO  (qtp476951373-6814) [n:127.0.0.1:59990_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 782462 INFO  (qtp476951373-6814) [n:127.0.0.1:59990_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 782470 INFO  (qtp476951373-6814) [n:127.0.0.1:59990_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=46, maxMergeAtOnceExplicit=23, maxMergedSegmentMB=27.3935546875, floorSegmentMB=1.404296875, forceMergeDeletesPctAllowed=20.937546942279567, segmentsPerTier=46.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6959952585937709
   [junit4]   2> 782470 INFO  (qtp476951373-6814) [n:127.0.0.1:59990_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.s.SolrIndexSearcher Opening [Searcher@59eb081c[collection1_shard1_replica_t43] main]
   [junit4]   2> 782472 INFO  (qtp476951373-6814) [n:127.0.0.1:59990_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 782473 INFO  (qtp476951373-6814) [n:127.0.0.1:59990_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 782474 INFO  (qtp476951373-6814) [n:127.0.0.1:59990_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 782474 INFO  (qtp476951373-6814) [n:127.0.0.1:59990_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1576070764624347136
   [junit4]   2> 782475 INFO  (searcherExecutor-2808-thread-1-processing-n:127.0.0.1:59990_ x:collection1_shard1_replica_t43 s:shard1 c:collection1) [n:127.0.0.1:59990_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.SolrCore [collection1_shard1_replica_t43] Registered new searcher Searcher@59eb081c[collection1_shard1_replica_t43] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 782486 INFO  (qtp476951373-6814) [n:127.0.0.1:59990_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 782487 INFO  (qtp476951373-6814) [n:127.0.0.1:59990_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 782487 INFO  (qtp476951373-6814) [n:127.0.0.1:59990_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:59990/collection1_shard1_replica_t43/
   [junit4]   2> 782487 INFO  (qtp476951373-6814) [n:127.0.0.1:59990_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 782487 INFO  (qtp476951373-6814) [n:127.0.0.1:59990_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.SyncStrategy http://127.0.0.1:59990/collection1_shard1_replica_t43/ has no replicas
   [junit4]   2> 782487 INFO  (qtp476951373-6814) [n:127.0.0.1:59990_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 782489 INFO  (qtp476951373-6814) [n:127.0.0.1:59990_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.ZkController collection1_shard1_replica_t43 stopping background replication from leader
   [junit4]   2> 782494 INFO  (qtp476951373-6814) [n:127.0.0.1:59990_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:59990/collection1_shard1_replica_t43/ shard1
   [junit4]   2> 782653 INFO  (qtp476951373-6814) [n:127.0.0.1:59990_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 782659 INFO  (qtp476951373-6814) [n:127.0.0.1:59990_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t43&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=1550
   [junit4]   2> 782663 INFO  (qtp476951373-6812) [n:127.0.0.1:59990_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:59990_&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2} status=0 QTime=1570
   [junit4]   2> 783099 INFO  (OverseerCollectionConfigSetProcessor-98504422271746052-127.0.0.1:59965_-n_0000000000) [n:127.0.0.1:59965_    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000006 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 783144 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_D6A2CC4B3E46D91F-001/shard-3-001 of type TLOG
   [junit4]   2> 783145 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 783145 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3fdf6842{/,null,AVAILABLE}
   [junit4]   2> 783145 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@1ac3983d{HTTP/1.1,[http/1.1]}{127.0.0.1:60003}
   [junit4]   2> 783145 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.e.j.s.Server Started @787743ms
   [junit4]   2> 783149 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_D6A2CC4B3E46D91F-001/tempDir-001/jetty3, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/, hostPort=60002, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_D6A2CC4B3E46D91F-001/shard-3-001/cores}
   [junit4]   2> 783150 ERROR (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 783151 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.1.0
   [junit4]   2> 783152 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 783152 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 783152 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-08-18T12:10:09.739Z
   [junit4]   2> 783156 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 783156 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_D6A2CC4B3E46D91F-001/shard-3-001/solr.xml
   [junit4]   2> 783166 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@235c5dfc, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 783183 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:59960/solr
   [junit4]   2> 783199 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [n:127.0.0.1:60002_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 783202 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [n:127.0.0.1:60002_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 783205 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [n:127.0.0.1:60002_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60002_
   [junit4]   2> 783208 INFO  (zkCallback-1077-thread-1-processing-n:127.0.0.1:59990_) [n:127.0.0.1:59990_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 783208 INFO  (zkCallback-1066-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 783208 INFO  (zkCallback-1071-thread-1-processing-n:127.0.0.1:59979_) [n:127.0.0.1:59979_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 783210 INFO  (zkCallback-1059-thread-1-processing-n:127.0.0.1:59965_) [n:127.0.0.1:59965_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 783212 INFO  (zkCallback-1083-thread-1-processing-n:127.0.0.1:60002_) [n:127.0.0.1:60002_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 783332 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [n:127.0.0.1:60002_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@235c5dfc
   [junit4]   2> 783343 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [n:127.0.0.1:60002_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@235c5dfc
   [junit4]   2> 783343 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [n:127.0.0.1:60002_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@235c5dfc
   [junit4]   2> 783345 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [n:127.0.0.1:60002_    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_D6A2CC4B3E46D91F-001/shard-3-001/cores
   [junit4]   2> 783419 INFO  (qtp476951373-6815) [n:127.0.0.1:59990_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:60002_&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 783423 INFO  (OverseerThreadFactory-2783-thread-5-processing-n:127.0.0.1:59965_) [n:127.0.0.1:59965_    ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:60002_ for creating new replica
   [junit4]   2> 783428 INFO  (SocketProxy-Acceptor-60002) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=60007,localport=60002], receiveBufferSize:408300
   [junit4]   2> 783433 INFO  (SocketProxy-Acceptor-60002) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=60003,localport=60008], receiveBufferSize=408300
   [junit4]   2> 783439 INFO  (qtp1337295756-6846) [n:127.0.0.1:60002_    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t45&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 783440 INFO  (qtp1337295756-6846) [n:127.0.0.1:60002_    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 784479 INFO  (qtp1337295756-6846) [n:127.0.0.1:60002_ c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 784516 INFO  (qtp1337295756-6846) [n:127.0.0.1:60002_ c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.s.IndexSchema [collection1_shard2_replica_t45] Schema name=test
   [junit4]   2> 784650 INFO  (qtp1337295756-6846) [n:127.0.0.1:60002_ c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 784681 INFO  (qtp1337295756-6846) [n:127.0.0.1:60002_ c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_t45' using configuration from collection collection1, trusted=true
   [junit4]   2> 784682 INFO  (qtp1337295756-6846) [n:127.0.0.1:60002_ c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_t45' (registry 'solr.core.collection1.shard2.replica_t45') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@235c5dfc
   [junit4]   2> 784682 INFO  (qtp1337295756-6846) [n:127.0.0.1:60002_ c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 784682 INFO  (qtp1337295756-6846) [n:127.0.0.1:60002_ c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.c.SolrCore [[collection1_shard2_replica_t45] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_D6A2CC4B3E46D91F-001/shard-3-001/cores/collection1_shard2_replica_t45], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_D6A2CC4B3E46D91F-001/shard-3-001/cores/collection1_shard2_replica_t45/data/]
   [junit4]   2> 784685 INFO  (qtp1337295756-6846) [n:127.0.0.1:60002_ c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1420090314, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 784688 WARN  (qtp1337295756-6846) [n:127.0.0.1:60002_ c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 784770 INFO  (qtp1337295756-6846) [n:127.0.0.1:60002_ c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 784771 INFO  (qtp1337295756-6846) [n:127.0.0.1:60002_ c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 784789 INFO  (qtp1337295756-6846) [n:127.0.0.1:60002_ c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 784789 INFO  (qtp1337295756-6846) [n:127.0.0.1:60002_ c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 784791 INFO  (qtp1337295756-6846) [n:127.0.0.1:60002_ c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=46, maxMergeAtOnceExplicit=23, maxMergedSegmentMB=27.3935546875, floorSegmentMB=1.404296875, forceMergeDeletesPctAllowed=20.937546942279567, segmentsPerTier=46.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6959952585937709
   [junit4]   2> 784791 INFO  (qtp1337295756-6846) [n:127.0.0.1:60002_ c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.s.SolrIndexSearcher Opening [Searcher@5eb25108[collection1_shard2_replica_t45] main]
   [junit4]   2> 784793 INFO  (qtp1337295756-6846) [n:127.0.0.1:60002_ c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 784794 INFO  (qtp1337295756-6846) [n:127.0.0.1:60002_ c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 784795 INFO  (qtp1337295756-6846) [n:127.0.0.1:60002_ c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 784795 INFO  (qtp1337295756-6846) [n:127.0.0.1:60002_ c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1576070767058092032
   [junit4]   2> 784796 INFO  (searcherExecutor-2819-thread-1-processing-n:127.0.0.1:60002_ x:collection1_shard2_replica_t45 s:shard2 c:collection1) [n:127.0.0.1:60002_ c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.c.SolrCore [collection1_shard2_replica_t45] Registered new searcher Searcher@5eb25108[collection1_shard2_replica_t45] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 784804 INFO  (qtp1337295756-6846) [n:127.0.0.1:60002_ c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.c.ZkController Core needs to recover:collection1_shard2_replica_t45
   [junit4]   2> 784804 INFO  (updateExecutor-1080-thread-1-processing-n:127.0.0.1:60002_ x:collection1_shard2_replica_t45 s:shard2 c:collection1) [n:127.0.0.1:60002_ c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.u.DefaultSolrCoreState Running recovery
   [junit4]   2> 784805 INFO  (recoveryExecutor-1081-thread-1-processing-n:127.0.0.1:60002_ x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:60002_ c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
   [junit4]   2> 784805 INFO  (qtp1337295756-6846) [n:127.0.0.1:60002_ c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t45&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=1366
   [junit4]   2> 784805 INFO  (recoveryExecutor-1081-thread-1-processing-n:127.0.0.1:60002_ x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:60002_ c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
   [junit4]   2> 784805 INFO  (recoveryExecutor-1081-thread-1-processing-n:127.0.0.1:60002_ x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:60002_ c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.c.ZkController collection1_shard2_replica_t45 stopping background replication from leader
   [junit4]   2> 784805 INFO  (recoveryExecutor-1081-thread-1-processing-n:127.0.0.1:60002_ x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:60002_ c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1_shard2_replica_t45]
   [junit4]   2> 784805 INFO  (recoveryExecutor-1081-thread-1-processing-n:127.0.0.1:60002_ x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:60002_ c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
   [junit4]   2> 784805 INFO  (recoveryExecutor-1081-thread-1-processing-n:127.0.0.1:60002_ x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:60002_ c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.c.RecoveryStrategy Publishing state of core [collection1_shard2_replica_t45] as recovering, leader is [http://127.0.0.1:59979/collection1_shard2_replica_t41/] and I am [http://127.0.0.1:60002/collection1_shard2_replica_t45/]
   [junit4]   2> 784809 INFO  (qtp476951373-6815) [n:127.0.0.1:59990_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:60002_&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2} status=0 QTime=1389
   [junit4]   2> 784810 INFO  (recoveryExecutor-1081-thread-1-processing-n:127.0.0.1:60002_ x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:60002_ c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:59979]; [WaitForState: action=PREPRECOVERY&core=collection1_shard2_replica_t41&nodeName=127.0.0.1:60002_&coreNodeName=core_node46&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
   [junit4]   2> 784811 INFO  (SocketProxy-Acceptor-59979) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=60011,localport=59979], receiveBufferSize:408300
   [junit4]   2> 784812 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.SolrTestCaseJ4 ###Starting test
   [junit4]   2> 784812 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Wait for recoveries to finish - wait 30000 for each attempt
   [junit4]   2> 784812 INFO  (TEST-HttpPartitionTest.test-seed#[D6A2CC4B3E46D91F]) [    ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: collection1 failOnTimeout:true timeout (sec):30000
   [junit4]   2> 784812 INFO  (SocketProxy-Acceptor-59979) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=59980,localport=60012], receiveBufferSize=408300
   [junit4]   2> 784813 INFO  (qtp1632235631-6782) [n:127.0.0.1:59979_    ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node46, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true, maxTime: 183 s
   [junit4]   2> 784818 INFO  (qtp1632235631-6782) [n:127.0.0.1:59979_    ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard2, thisCore=collection1_shard2_replica_t41, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:60002_, coreNodeName=core_node46, onlyIfActiveCheckResult=false, nodeProps: core_node46:{"core":"collection1_shard2_replica_t45","base_url":"http://127.0.0.1:60002","node_name":"127.0.0.1:60002_","state":"down","type":"TLOG"}
   [junit4]   2> 785425 INFO  (OverseerCollectionConfigSetProcessor-98504422271746052-127.0.0.1:59965_-n_0000000000) [n:127.0.0.1:59965_    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000008 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 785825 INFO  (qtp1632235631-6782) [n:127.0.0.1:59979_    ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard2, thisCore=collection1_shard2_replica_t41, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:60002_, coreNodeName=core_node46, onlyIfActiveCheckResult=false, nodeProps: core_node46:{"core":"collection1_shard2_replica_t45","base_url":"http://127.0.0.1:60002","node_name":"127.0.0.1:60002_","state":"recovering","type":"TLOG"}
   [junit4]   2> 785825 INFO  (qtp1632235631-6782) [n:127.0.0.1:59979_    ] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node46, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
   [junit4]   2> 785825 INFO  (qtp1632235631-6782) [n:127.0.0.1:59979_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:60002_&onlyIfLeaderActive=true&core=collection1_shard2_replica_t41&coreNodeName=core_node46&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=1011
   [junit4]   2> 786327 INFO  (recoveryExecutor-1081-thread-1-processing-n:127.0.0.1:60002_ x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:60002_ c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.c.RecoveryStrategy Starting Replication Recovery.
   [junit4]   2> 786327 INFO  (recoveryExecutor-1081-thread-1-processing-n:127.0.0.1:60002_ x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:60002_ c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.c.RecoveryStrategy Attempting to replicate from [http://127.0.0.1:59979/collection1_shard2_replica_t41/].
   [junit4]   2> 786328 INFO  (SocketProxy-Acceptor-59979) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=60013,localport=59979], receiveBufferSize:408300
   [junit4]   2> 786329 INFO  (SocketProxy-Acceptor-59979) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=59980,localport=60014], receiveBufferSize=408300
   [junit4]   2> 786335 INFO  (qtp1632235631-6779) [n:127.0.0.1:59979_ c:collection1 s:shard2 r:core

[...truncated too long message...]

2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	45	/solr/collections/tlog_replica_test_recovery/state.json
   [junit4]   2> 	28	/solr/collections/tlog_replica_test_remove_leader/state.json
   [junit4]   2> 	27	/solr/collections/tlog_replica_test_add_remove_tlog_replica/state.json
   [junit4]   2> 	26	/solr/collections/tlog_replica_test_kill_leader/state.json
   [junit4]   2> 	26	/solr/collections/tlog_replica_test_create_delete/state.json
   [junit4]   2> 	24	/solr/collections/tlog_replica_test_kill_tlog_replica/state.json
   [junit4]   2> 	22	/solr/collections/tlog_replica_test_basic_leader_election/state.json
   [junit4]   2> 	15	/solr/collections/tlog_replica_test_only_leader_indexes/state.json
   [junit4]   2> 	15	/solr/collections/tlog_replica_test_delete_by_id/state.json
   [junit4]   2> 	13	/solr/collections/tlog_replica_test_out_of_order_db_qwith_in_place_updates/state.json
   [junit4]   2> 	13	/solr/collections/tlog_replica_test_real_time_get/state.json
   [junit4]   2> 	9	/solr/clusterprops.json
   [junit4]   2> 	9	/solr/clusterstate.json
   [junit4]   2> 	6	/solr/collections/tlog_replica_test_add_docs/state.json
   [junit4]   2> 	4	/solr/collections/tlog_replica_test_recovery/leader_elect/shard1/election/98504556168216590-core_node3-n_0000000000
   [junit4]   2> 	3	/solr/overseer_elect/election/98504556168216590-127.0.0.1:54140_solr-n_0000000003
   [junit4]   2> 	2	/solr/collections/tlog_replica_test_kill_tlog_replica/leader_elect/shard1/election/98504556168216587-core_node4-n_0000000000
   [junit4]   2> 	2	/solr/collections/tlog_replica_test_create_delete/leader_elect/shard2/election/98504556168216581-core_node13-n_0000000000
   [junit4]   2> 	2	/solr/collections/tlog_replica_test_create_delete/leader_elect/shard1/election/98504556168216582-core_node7-n_0000000000
   [junit4]   2> 	2	/solr/overseer_elect/election/98504556168216582-127.0.0.1:54140_solr-n_0000000000
   [junit4]   2> 	2	/solr/collections/tlog_replica_test_create_delete/leader_elect/shard1/election/98504556168216581-core_node9-n_0000000000
   [junit4]   2> 	2	/solr/collections/tlog_replica_test_create_delete/leader_elect/shard1/election/98504556168216581-core_node9-n_0000000001
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	9	/solr/collections
   [junit4]   2> 	4	/solr/overseer/queue
   [junit4]   2> 	4	/solr/overseer/collection-queue-work
   [junit4]   2> 	3	/solr/live_nodes
   [junit4]   2> 
   [junit4]   2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestTlogReplica_D6A2CC4B3E46D91F-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {foo=FST50, id=PostingsFormat(name=LuceneFixedGap), foo_s=FST50}, docValues:{_version_=DocValuesFormat(name=Direct), id=DocValuesFormat(name=Lucene70)}, maxPointsInLeafNode=73, maxMBSortInHeap=7.42102680662157, sim=RandomSimilarity(queryNorm=true): {}, locale=en-MT, timezone=Asia/Aden
   [junit4]   2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=3,threads=1,free=259247576,total=518979584
   [junit4]   2> NOTE: All tests run in this JVM: [TestFiltering, TestCopyFieldCollectionResource, QueryResultKeyTest, AsyncCallRequestStatusResponseTest, MoreLikeThisHandlerTest, TestRawTransformer, TestDocSet, TestXmlQParser, DocumentBuilderTest, TestConfigsApi, TestObjectReleaseTracker, HttpSolrCallGetCoreTest, TestJettySolrRunner, BadCopyFieldTest, TestCollapseQParserPlugin, TestAuthorizationFramework, BlockDirectoryTest, TestRandomRequestDistribution, TestSolr4Spatial2, TestPrepRecovery, TestHttpShardHandlerFactory, LeaderElectionContextKeyTest, TestRandomCollapseQParserPlugin, DocValuesMultiTest, DistributedQueryComponentCustomSortTest, TestMaxScoreQueryParser, CacheHeaderTest, TestLegacyField, TestIntervalFaceting, ConjunctionSolrSpellCheckerTest, TestBinaryResponseWriter, TestTrie, ChaosMonkeySafeLeaderTest, OutOfBoxZkACLAndCredentialsProvidersTest, TestCoreDiscovery, TestSchemaManager, TestDocumentBuilder, TestCSVLoader, LeaderFailureAfterFreshStartTest, ChangedSchemaMergeTest, ShufflingReplicaListTransformerTest, EchoParamsTest, DistributedFacetPivotLongTailTest, UpdateParamsTest, DeleteNodeTest, DeleteInactiveReplicaTest, SpellingQueryConverterTest, TestCloudDeleteByQuery, TestRandomFaceting, TestLMJelinekMercerSimilarityFactory, SortSpecParsingTest, SpatialRPTFieldTypeTest, InfoHandlerTest, TestJsonFacetRefinement, DirectSolrSpellCheckerTest, TestFileDictionaryLookup, JsonLoaderTest, TermVectorComponentTest, FileUtilsTest, HdfsRecoveryZkTest, SOLR749Test, AutoCommitTest, RuleEngineTest, TestJsonRequest, StatsComponentTest, CursorMarkTest, TestWordDelimiterFilterFactory, V2StandaloneTest, SolrXmlInZkTest, VMParamsZkACLAndCredentialsProvidersTest, TestPostingsSolrHighlighter, PreAnalyzedFieldManagedSchemaCloudTest, SolrMetricsIntegrationTest, TestCollationField, TestSolr4Spatial, TestConfigSetProperties, TestSQLHandler, TestSizeLimitedDistributedMap, SolrRequestParserTest, TestReload, TestLRUCache, BlobRepositoryCloudTest, TestMiniSolrCloudClusterSSL, TestUniqueKeyFieldResource, SuggestComponentContextFilterQueryTest, DebugComponentTest, TestSchemaResource, TestSchemaVersionResource, MBeansHandlerTest, TolerantUpdateProcessorTest, TestAnalyzeInfixSuggestions, SignatureUpdateProcessorFactoryTest, TestSystemCollAutoCreate, TestFieldSortValues, TestSolrJ, TlogReplayBufferedWhileIndexingTest, TestRealTimeGet, SolrIndexMetricsTest, TestLegacyNumericUtils, TermVectorComponentDistributedTest, AnalysisErrorHandlingTest, DataDrivenBlockJoinTest, TestConfigSetsAPIZkFailure, TestExactSharedStatsCache, HighlighterMaxOffsetTest, TestCloudManagedSchema, TestSolrIndexConfig, TestCorePropertiesReload, AssignBackwardCompatibilityTest, TemplateUpdateProcessorTest, LeaderInitiatedRecoveryOnShardRestartTest, TestRangeQuery, TestDynamicLoading, TestStressUserVersions, ExitableDirectoryReaderTest, TestDeleteCollectionOnDownNodes, TestInPlaceUpdatesDistrib, TestBM25SimilarityFactory, ShowFileRequestHandlerTest, TestLuceneMatchVersion, TestMacroExpander, RecoveryAfterSoftCommitTest, TestStressLiveNodes, HdfsWriteToMultipleCollectionsTest, CollectionsAPIDistributedZkTest, FileBasedSpellCheckerTest, TestElisionMultitermQuery, TestUseDocValuesAsStored, DistributedTermsComponentTest, TestSuggestSpellingConverter, TestChildDocTransformer, TestApiFramework, ReplaceNodeTest, TestExportWriter, TestLeaderElectionZkExpiry, TestFieldCacheReopen, SolrPluginUtilsTest, TestInfoStreamLogging, IgnoreCommitOptimizeUpdateProcessorFactoryTest, CreateCollectionCleanupTest, RestartWhileUpdatingTest, TestMultiValuedNumericRangeQuery, SpellCheckComponentTest, CollectionReloadTest, TestCloudPseudoReturnFields, PropertiesRequestHandlerTest, TestCloudJSONFacetJoinDomain, BigEndianAscendingWordDeserializerTest, LukeRequestHandlerTest, MissingSegmentRecoveryTest, CdcrVersionReplicationTest, SynonymTokenizerTest, BasicFunctionalityTest, TestScoreJoinQPNoScore, TestEmbeddedSolrServerConstructors, TestTolerantUpdateProcessorCloud, TestJoin, TestSolrConfigHandlerCloud, TestUninvertingReader, CoreSorterTest, BadComponentTest, TestReRankQParserPlugin, TestManagedSynonymFilterFactory, BlockJoinFacetDistribTest, TermsComponentTest, DistributedVersionInfoTest, HdfsChaosMonkeySafeLeaderTest, ZkStateWriterTest, DistributedQueryElevationComponentTest, TestShardHandlerFactory, TestRestManager, ShardSplitTest, TestSegmentSorting, TestSolrQueryParser, TestDelegationWithHadoopAuth, TestExceedMaxTermLength, TestRecovery, TestHighFrequencyDictionaryFactory, BasicDistributedZk2Test, UnloadDistributedZkTest, SyncSliceTest, BasicZkTest, RecoveryZkTest, TestReplicationHandler, TestRandomDVFaceting, TestFaceting, ZkControllerTest, SolrCoreTest, TestGroupingSearch, QueryElevationComponentTest, BadIndexSchemaTest, ConvertedLegacyTest, TestIndexSearcher, SoftAutoCommitTest, TestQueryUtils, DocumentAnalysisRequestHandlerTest, TestQueryTypes, PrimitiveFieldTypeTest, RequestHandlersTest, PathHierarchyTokenizerFactoryTest, IndexSchemaRuntimeFieldTest, UpdateRequestProcessorFactoryTest, QueryParsingTest, UniqFieldsUpdateProcessorFactoryTest, PingRequestHandlerTest, HighlighterConfigTest, CopyFieldTest, TestStressRecovery, NumericFieldsTest, TestConfig, ExternalFileFieldSortTest, TestSolrCoreProperties, SpellPossibilityIteratorTest, TestCharFilters, TestPerFieldSimilarity, TestFastWriter, ResourceLoaderTest, ScriptEngineTest, URLClassifyProcessorTest, PluginInfoTest, PreAnalyzedFieldTest, RAMDirectoryFactoryTest, TestUtils, SystemInfoHandlerTest, CircularListTest, TestRTGBase, DistributedIntervalFacetingTest, TestHighlightDedupGrouping, TestTolerantSearch, ActionThrottleTest, CdcrBootstrapTest, ChaosMonkeySafeLeaderWithPullReplicasTest, CloudExitableDirectoryReaderTest, CollectionTooManyReplicasTest, CollectionsAPISolrJTest, DistribDocExpirationUpdateProcessorTest, MigrateRouteKeyTest, MoveReplicaHDFSTest, MoveReplicaHDFSUlogDirTest, MultiThreadedOCPTest, NodeMutatorTest, OverriddenZkACLAndCredentialsProvidersTest, OverseerCollectionConfigSetProcessorTest, OverseerRolesTest, OverseerStatusTest, OverseerTaskQueueTest, RemoteQueryErrorTest, ReplicationFactorTest, SSLMigrationTest, SharedFSAutoReplicaFailoverTest, SharedFSAutoReplicaFailoverUtilsTest, SolrCLIZkUtilsTest, SolrCloudExampleTest, TestCloudSearcherWarming, TestCryptoKeys, TestDistribDocBasedVersion, TestDownShardTolerantSearch, TestExclusionRuleCollectionAccess, TestHdfsCloudBackupRestore, TestLeaderInitiatedRecoveryThread, TestLocalFSCloudBackupRestore, TestPullReplicaErrorHandling, TestRebalanceLeaders, TestRequestForwarding, TestRequestStatusCollectionAPI, TestSolrCloudWithDelegationTokens, TestSolrCloudWithKerberosAlt, TestStressCloudBlindAtomicUpdates, TestTlogReplica]
   [junit4] Completed [547/729 (2!)] on J1 in 101.94s, 13 tests, 2 errors <<< FAILURES!

[...truncated 44041 lines...]

[JENKINS] Lucene-Solr-7.x-MacOSX (64bit/jdk1.8.0) - Build # 124 - Still unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-MacOSX/124/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseParallelGC

18 tests failed.
FAILED:  org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test

Error Message:
Could not find collection:collection2

Stack Trace:
java.lang.AssertionError: Could not find collection:collection2
	at __randomizedtesting.SeedInfo.seed([9FCE348E3E843E85:179A0B549078537D]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNotNull(Assert.java:526)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:155)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:140)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:135)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:908)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.testIndexingBatchPerRequestWithHttpSolrClient(FullSolrCloudDistribCmdsTest.java:612)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test(FullSolrCloudDistribCmdsTest.java:152)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity

Error Message:
IOException occured when talking to server at: http://127.0.0.1:53324/solr/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:53324/solr/collection1
	at __randomizedtesting.SeedInfo.seed([9FCE348E3E843E85:33D1FA941101D35D]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:641)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:173)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:138)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:152)
	at org.apache.solr.handler.TestReplicationHandler.index(TestReplicationHandler.java:180)
	at org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity(TestReplicationHandler.java:1185)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:53324 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:526)
	... 47 more


FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestStopPoll

Error Message:
/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_9FCE348E3E843E85-001/solr-instance-004/collection1/core.properties: Too many open files

Stack Trace:
java.nio.file.FileSystemException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_9FCE348E3E843E85-001/solr-instance-004/collection1/core.properties: Too many open files
	at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91)
	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107)
	at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214)
	at java.nio.file.spi.FileSystemProvider.newOutputStream(FileSystemProvider.java:434)
	at java.nio.file.Files.newOutputStream(Files.java:216)
	at org.apache.solr.SolrTestCaseJ4.writeCoreProperties(SolrTestCaseJ4.java:197)
	at org.apache.solr.handler.TestReplicationHandler$SolrInstance.setUp(TestReplicationHandler.java:1628)
	at org.apache.solr.handler.TestReplicationHandler.setUp(TestReplicationHandler.java:123)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:941)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestIllegalFilePaths

Error Message:
/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_9FCE348E3E843E85-001/solr-instance-006/collection1/core.properties: Too many open files

Stack Trace:
java.nio.file.FileSystemException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_9FCE348E3E843E85-001/solr-instance-006/collection1/core.properties: Too many open files
	at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91)
	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107)
	at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214)
	at java.nio.file.spi.FileSystemProvider.newOutputStream(FileSystemProvider.java:434)
	at java.nio.file.Files.newOutputStream(Files.java:216)
	at org.apache.solr.SolrTestCaseJ4.writeCoreProperties(SolrTestCaseJ4.java:197)
	at org.apache.solr.handler.TestReplicationHandler$SolrInstance.setUp(TestReplicationHandler.java:1628)
	at org.apache.solr.handler.TestReplicationHandler.setUp(TestReplicationHandler.java:123)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:941)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterStartup

Error Message:
/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_9FCE348E3E843E85-001/solr-instance-008/collection1/core.properties: Too many open files

Stack Trace:
java.nio.file.FileSystemException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_9FCE348E3E843E85-001/solr-instance-008/collection1/core.properties: Too many open files
	at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91)
	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107)
	at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214)
	at java.nio.file.spi.FileSystemProvider.newOutputStream(FileSystemProvider.java:434)
	at java.nio.file.Files.newOutputStream(Files.java:216)
	at org.apache.solr.SolrTestCaseJ4.writeCoreProperties(SolrTestCaseJ4.java:197)
	at org.apache.solr.handler.TestReplicationHandler$SolrInstance.setUp(TestReplicationHandler.java:1628)
	at org.apache.solr.handler.TestReplicationHandler.setUp(TestReplicationHandler.java:123)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:941)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.handler.TestReplicationHandler.testRateLimitedReplication

Error Message:
/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_9FCE348E3E843E85-001/solr-instance-010/collection1/core.properties: Too many open files

Stack Trace:
java.nio.file.FileSystemException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_9FCE348E3E843E85-001/solr-instance-010/collection1/core.properties: Too many open files
	at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91)
	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107)
	at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214)
	at java.nio.file.spi.FileSystemProvider.newOutputStream(FileSystemProvider.java:434)
	at java.nio.file.Files.newOutputStream(Files.java:216)
	at org.apache.solr.SolrTestCaseJ4.writeCoreProperties(SolrTestCaseJ4.java:197)
	at org.apache.solr.handler.TestReplicationHandler$SolrInstance.setUp(TestReplicationHandler.java:1628)
	at org.apache.solr.handler.TestReplicationHandler.setUp(TestReplicationHandler.java:123)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:941)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.handler.TestReplicationHandler.testEmptyCommits

Error Message:
/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_9FCE348E3E843E85-001/solr-instance-012/collection1/core.properties: Too many open files

Stack Trace:
java.nio.file.FileSystemException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_9FCE348E3E843E85-001/solr-instance-012/collection1/core.properties: Too many open files
	at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91)
	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107)
	at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214)
	at java.nio.file.spi.FileSystemProvider.newOutputStream(FileSystemProvider.java:434)
	at java.nio.file.Files.newOutputStream(Files.java:216)
	at org.apache.solr.SolrTestCaseJ4.writeCoreProperties(SolrTestCaseJ4.java:197)
	at org.apache.solr.handler.TestReplicationHandler$SolrInstance.setUp(TestReplicationHandler.java:1628)
	at org.apache.solr.handler.TestReplicationHandler.setUp(TestReplicationHandler.java:123)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:941)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestIndexAndConfigReplication

Error Message:
Error from server at http://127.0.0.1:53331/solr/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 404 </title> </head> <body> <h2>HTTP ERROR: 404</h2> <p>Problem accessing /solr/collection1/select. Reason: <pre>    javax.servlet.UnavailableException: Error processing the request. CoreContainer is either not initialized or shutting down.</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.20.v20170531</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:53331/solr/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 404 </title>
</head>
<body>
<h2>HTTP ERROR: 404</h2>
<p>Problem accessing /solr/collection1/select. Reason:
<pre>    javax.servlet.UnavailableException: Error processing the request. CoreContainer is either not initialized or shutting down.</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.20.v20170531</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([9FCE348E3E843E85:8B866FDB1D83839B]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:591)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:957)
	at org.apache.solr.handler.TestReplicationHandler.query(TestReplicationHandler.java:189)
	at org.apache.solr.handler.TestReplicationHandler.clearIndexWithReplication(TestReplicationHandler.java:131)
	at org.apache.solr.handler.TestReplicationHandler.doTestIndexAndConfigReplication(TestReplicationHandler.java:498)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestIndexAndConfigAliasReplication

Error Message:
/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_9FCE348E3E843E85-001/solr-instance-016/collection1/core.properties: Too many open files

Stack Trace:
java.nio.file.FileSystemException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_9FCE348E3E843E85-001/solr-instance-016/collection1/core.properties: Too many open files
	at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91)
	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107)
	at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214)
	at java.nio.file.spi.FileSystemProvider.newOutputStream(FileSystemProvider.java:434)
	at java.nio.file.Files.newOutputStream(Files.java:216)
	at org.apache.solr.SolrTestCaseJ4.writeCoreProperties(SolrTestCaseJ4.java:197)
	at org.apache.solr.handler.TestReplicationHandler$SolrInstance.setUp(TestReplicationHandler.java:1628)
	at org.apache.solr.handler.TestReplicationHandler.setUp(TestReplicationHandler.java:123)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:941)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload

Error Message:
/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_9FCE348E3E843E85-001/solr-instance-018/collection1/core.properties: Too many open files

Stack Trace:
java.nio.file.FileSystemException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_9FCE348E3E843E85-001/solr-instance-018/collection1/core.properties: Too many open files
	at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91)
	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107)
	at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214)
	at java.nio.file.spi.FileSystemProvider.newOutputStream(FileSystemProvider.java:434)
	at java.nio.file.Files.newOutputStream(Files.java:216)
	at org.apache.solr.SolrTestCaseJ4.writeCoreProperties(SolrTestCaseJ4.java:197)
	at org.apache.solr.handler.TestReplicationHandler$SolrInstance.setUp(TestReplicationHandler.java:1628)
	at org.apache.solr.handler.TestReplicationHandler.setUp(TestReplicationHandler.java:123)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:941)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestDetails

Error Message:
/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_9FCE348E3E843E85-001/solr-instance-020/collection1/core.properties: Too many open files

Stack Trace:
java.nio.file.FileSystemException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_9FCE348E3E843E85-001/solr-instance-020/collection1/core.properties: Too many open files
	at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91)
	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107)
	at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214)
	at java.nio.file.spi.FileSystemProvider.newOutputStream(FileSystemProvider.java:434)
	at java.nio.file.Files.newOutputStream(Files.java:216)
	at org.apache.solr.SolrTestCaseJ4.writeCoreProperties(SolrTestCaseJ4.java:197)
	at org.apache.solr.handler.TestReplicationHandler$SolrInstance.setUp(TestReplicationHandler.java:1628)
	at org.apache.solr.handler.TestReplicationHandler.setUp(TestReplicationHandler.java:123)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:941)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterWrite2Slave

Error Message:
/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_9FCE348E3E843E85-001/solr-instance-022/collection1/core.properties: Too many open files

Stack Trace:
java.nio.file.FileSystemException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_9FCE348E3E843E85-001/solr-instance-022/collection1/core.properties: Too many open files
	at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91)
	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107)
	at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214)
	at java.nio.file.spi.FileSystemProvider.newOutputStream(FileSystemProvider.java:434)
	at java.nio.file.Files.newOutputStream(Files.java:216)
	at org.apache.solr.SolrTestCaseJ4.writeCoreProperties(SolrTestCaseJ4.java:197)
	at org.apache.solr.handler.TestReplicationHandler$SolrInstance.setUp(TestReplicationHandler.java:1628)
	at org.apache.solr.handler.TestReplicationHandler.setUp(TestReplicationHandler.java:123)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:941)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestStressReplication

Error Message:
/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_9FCE348E3E843E85-001/solr-instance-024/collection1/core.properties: Too many open files

Stack Trace:
java.nio.file.FileSystemException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_9FCE348E3E843E85-001/solr-instance-024/collection1/core.properties: Too many open files
	at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91)
	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107)
	at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214)
	at java.nio.file.spi.FileSystemProvider.newOutputStream(FileSystemProvider.java:434)
	at java.nio.file.Files.newOutputStream(Files.java:216)
	at org.apache.solr.SolrTestCaseJ4.writeCoreProperties(SolrTestCaseJ4.java:197)
	at org.apache.solr.handler.TestReplicationHandler$SolrInstance.setUp(TestReplicationHandler.java:1628)
	at org.apache.solr.handler.TestReplicationHandler.setUp(TestReplicationHandler.java:123)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:941)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestIndexFetchWithMasterUrl

Error Message:
/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_9FCE348E3E843E85-001/solr-instance-026/collection1/core.properties: Too many open files

Stack Trace:
java.nio.file.FileSystemException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_9FCE348E3E843E85-001/solr-instance-026/collection1/core.properties: Too many open files
	at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91)
	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107)
	at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214)
	at java.nio.file.spi.FileSystemProvider.newOutputStream(FileSystemProvider.java:434)
	at java.nio.file.Files.newOutputStream(Files.java:216)
	at org.apache.solr.SolrTestCaseJ4.writeCoreProperties(SolrTestCaseJ4.java:197)
	at org.apache.solr.handler.TestReplicationHandler$SolrInstance.setUp(TestReplicationHandler.java:1628)
	at org.apache.solr.handler.TestReplicationHandler.setUp(TestReplicationHandler.java:123)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:941)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestRepeater

Error Message:
/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_9FCE348E3E843E85-001/solr-instance-028/collection1/core.properties: Too many open files

Stack Trace:
java.nio.file.FileSystemException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_9FCE348E3E843E85-001/solr-instance-028/collection1/core.properties: Too many open files
	at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91)
	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107)
	at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214)
	at java.nio.file.spi.FileSystemProvider.newOutputStream(FileSystemProvider.java:434)
	at java.nio.file.Files.newOutputStream(Files.java:216)
	at org.apache.solr.SolrTestCaseJ4.writeCoreProperties(SolrTestCaseJ4.java:197)
	at org.apache.solr.handler.TestReplicationHandler$SolrInstance.setUp(TestReplicationHandler.java:1628)
	at org.apache.solr.handler.TestReplicationHandler.setUp(TestReplicationHandler.java:123)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:941)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestHandlerPathUnchanged

Error Message:
/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_9FCE348E3E843E85-001/solr-instance-030/collection1/core.properties: Too many open files

Stack Trace:
java.nio.file.FileSystemException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_9FCE348E3E843E85-001/solr-instance-030/collection1/core.properties: Too many open files
	at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91)
	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107)
	at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214)
	at java.nio.file.spi.FileSystemProvider.newOutputStream(FileSystemProvider.java:434)
	at java.nio.file.Files.newOutputStream(Files.java:216)
	at org.apache.solr.SolrTestCaseJ4.writeCoreProperties(SolrTestCaseJ4.java:197)
	at org.apache.solr.handler.TestReplicationHandler$SolrInstance.setUp(TestReplicationHandler.java:1628)
	at org.apache.solr.handler.TestReplicationHandler.setUp(TestReplicationHandler.java:123)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:941)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.handler.TestReplicationHandler

Error Message:
21 threads leaked from SUITE scope at org.apache.solr.handler.TestReplicationHandler:     1) Thread[id=17863, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicationHandler]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=17841, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicationHandler]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=17741, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicationHandler]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=17885, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicationHandler]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=17820, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicationHandler]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=17690, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicationHandler]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)    7) Thread[id=17689, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicationHandler]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)    8) Thread[id=17752, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicationHandler]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)    9) Thread[id=17763, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicationHandler]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)   10) Thread[id=17718, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicationHandler]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)   11) Thread[id=17809, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicationHandler]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)   12) Thread[id=17874, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicationHandler]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)   13) Thread[id=17830, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicationHandler]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)   14) Thread[id=17852, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicationHandler]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)   15) Thread[id=17706, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicationHandler]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)   16) Thread[id=17719, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicationHandler]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)   17) Thread[id=17730, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicationHandler]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)   18) Thread[id=17808, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicationHandler]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)   19) Thread[id=17717, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicationHandler]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)   20) Thread[id=17807, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicationHandler]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)   21) Thread[id=17705, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicationHandler]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 21 threads leaked from SUITE scope at org.apache.solr.handler.TestReplicationHandler: 
   1) Thread[id=17863, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicationHandler]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=17841, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicationHandler]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=17741, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicationHandler]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=17885, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicationHandler]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=17

[...truncated too long message...]

 o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 2952179 INFO  (jetty-closer-5465-thread-4) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@671f56f1: rootName = solr_57906, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@22daca6
   [junit4]   2> 2952182 INFO  (jetty-closer-5465-thread-4) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 2952190 WARN  (zkCallback-5484-thread-3-processing-n:127.0.0.1:57906_solr) [n:127.0.0.1:57906_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 2952191 INFO  (jetty-closer-5465-thread-4) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@1eadd03c{/solr,null,UNAVAILABLE}
   [junit4]   2> 2952192 ERROR (SUITE-TestSolrCloudSnapshots-seed#[9FCE348E3E843E85]-worker) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 2952193 INFO  (SUITE-TestSolrCloudSnapshots-seed#[9FCE348E3E843E85]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:57904 57904
   [junit4]   2> 2952238 INFO  (Thread-4789) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:57904 57904
   [junit4]   2> 2952870 WARN  (Thread-4789) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	5	/solr/aliases.json
   [junit4]   2> 	5	/solr/clusterprops.json
   [junit4]   2> 	4	/solr/security.json
   [junit4]   2> 	4	/solr/collections/SolrCloudSnapshots/state.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	12	/solr/collections/SolrCloudSnapshots/state.json
   [junit4]   2> 	5	/solr/clusterstate.json
   [junit4]   2> 	2	/solr/overseer_elect/election/98518266440384520-127.0.0.1:57907_solr-n_0000000000
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	5	/solr/live_nodes
   [junit4]   2> 	5	/solr/collections
   [junit4]   2> 
   [junit4]   2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.core.snapshots.TestSolrCloudSnapshots_9FCE348E3E843E85-001
   [junit4]   2> Aug 20, 2017 10:51:20 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarity(queryNorm=false): {}, locale=bg, timezone=Asia/Chita
   [junit4]   2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=3,threads=1,free=43816928,total=529530880
   [junit4]   2> NOTE: All tests run in this JVM: [TestPullReplicaErrorHandling, TestDynamicLoading, BadComponentTest, SSLMigrationTest, TestSolrCloudWithHadoopAuthPlugin, TestRTGBase, TestConfigSetsAPI, TestReloadDeadlock, TestDFISimilarityFactory, TermsComponentTest, TestRangeQuery, TestTrackingShardHandlerFactory, TestLuceneMatchVersion, SpellCheckCollatorTest, DistributedFacetPivotSmallTest, TestCrossCoreJoin, TestLegacyField, AssignBackwardCompatibilityTest, TestEmbeddedSolrServerConstructors, TestAnalyzeInfixSuggestions, DistributedQueryComponentCustomSortTest, SimpleMLTQParserTest, TestRandomCollapseQParserPlugin, TestComplexPhraseQParserPlugin, TestInitQParser, SolrMetricManagerTest, ChangedSchemaMergeTest, CoreSorterTest, TestConfigsApi, JsonLoaderTest, MetricsConfigTest, CloneFieldUpdateProcessorFactoryTest, TestDistributedGrouping, InfixSuggestersTest, DateRangeFieldTest, CollectionTooManyReplicasTest, TestHdfsBackupRestoreCore, TestPartialUpdateDeduplication, LeaderInitiatedRecoveryOnShardRestartTest, TestBM25SimilarityFactory, TestHighFrequencyDictionaryFactory, TestReRankQParserPlugin, CoreAdminRequestStatusTest, TestLeaderElectionZkExpiry, TestPerFieldSimilarity, TestRemoteStreaming, TestStressCloudBlindAtomicUpdates, SpatialRPTFieldTypeTest, TestUniqueKeyFieldResource, RestartWhileUpdatingTest, SpellCheckComponentTest, IndexSchemaRuntimeFieldTest, TestSolrCloudWithKerberosAlt, TestBlobHandler, TestValueSourceCache, TestPolicyCloud, SolrGraphiteReporterTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, CopyFieldTest, SearchHandlerTest, HdfsTlogReplayBufferedWhileIndexingTest, TestLocalFSCloudBackupRestore, WrapperMergePolicyFactoryTest, TestGraphTermsQParserPlugin, ExitableDirectoryReaderTest, DataDrivenBlockJoinTest, TestSortByMinMaxFunction, DocumentBuilderTest, TestConfigOverlay, TestMultiWordSynonyms, TestManagedResource, SolrCLIZkUtilsTest, TermVectorComponentDistributedTest, SubstringBytesRefFilterTest, TestSchemaManager, QueryEqualityTest, HdfsSyncSliceTest, CdcrBootstrapTest, ConfigSetsAPITest, TestDistributedMissingSort, AtomicUpdatesTest, PropertiesRequestHandlerTest, ConvertedLegacyTest, TestCloudManagedSchema, ConjunctionSolrSpellCheckerTest, TestFiltering, TestSuggestSpellingConverter, TestClusterStateMutator, TestJsonFacetRefinement, TestLRUCache, TestJsonRequest, TestReload, DeleteNodeTest, TestDocSet, TestSlowCompositeReaderWrapper, TestRealTimeGet, TestCloudSearcherWarming, FileBasedSpellCheckerTest, TestSolrCLIRunExample, MissingSegmentRecoveryTest, TestLRUStatsCache, CollectionsAPIDistributedZkTest, TestSimpleTrackingShardHandler, TestDistributedStatsComponentCardinality, TestDelegationWithHadoopAuth, LeaderElectionIntegrationTest, TestGraphMLResponseWriter, TestNumericTerms32, TestMultiValuedNumericRangeQuery, CollectionReloadTest, TestMinMaxOnMultiValuedField, TestSQLHandler, InfoHandlerTest, TlogReplayBufferedWhileIndexingTest, DistributedFacetExistsSmallTest, TestObjectReleaseTracker, CollectionsAPISolrJTest, ZkSolrClientTest, BasicDistributedZk2Test, DOMUtilTest, TestFastOutputStream, RequestLoggingTest, TestSimpleQParserPlugin, TestSolrIndexConfig, TestShardHandlerFactory, TermVectorComponentTest, TestQueryUtils, MetricUtilsTest, UpdateRequestProcessorFactoryTest, TestCoreDiscovery, TestRuleBasedAuthorizationPlugin, CursorMarkTest, SuggesterTSTTest, TestPerFieldSimilarityWithDefaultOverride, CacheHeaderTest, OverseerCollectionConfigSetProcessorTest, SolrIndexMetricsTest, TestIndexSearcher, TestUninvertingReader, LoggingHandlerTest, TestHdfsCloudBackupRestore, TestChildDocTransformer, TestScoreJoinQPNoScore, TestAnalyzedSuggestions, BigEndianAscendingWordDeserializerTest, PreAnalyzedFieldManagedSchemaCloudTest, SolrPluginUtilsTest, TestJoin, DistribJoinFromCollectionTest, TestPrepRecovery, ForceLeaderTest, DocumentAnalysisRequestHandlerTest, TestElisionMultitermQuery, BlockJoinFacetSimpleTest, SolrCloudReportersTest, OverseerTaskQueueTest, DeleteShardTest, HttpSolrCallGetCoreTest, DistribCursorPagingTest, TestCharFilters, TestConfigSetProperties, UpdateLogTest, BitVectorTest, TestCloudNestedDocsSort, TestCollationFieldDocValues, TemplateUpdateProcessorTest, FullHLLTest, TestApiFramework, TestSizeLimitedDistributedMap, ShufflingReplicaListTransformerTest, EnumFieldTest, TestCorePropertiesReload, ZkNodePropsTest, TestXmlQParser, TestTolerantUpdateProcessorCloud, HighlighterMaxOffsetTest, HdfsBasicDistributedZkTest, AnalyticsMergeStrategyTest, TestSolrConfigHandler, TestComponentsName, TestZkChroot, ChaosMonkeySafeLeaderTest, UnloadDistributedZkTest, OpenCloseCoreStressTest, OverseerTest, ShardRoutingTest, BasicZkTest, FullSolrCloudDistribCmdsTest, TestReplicationHandler, TestRandomFaceting, TestRandomDVFaceting, ShardRoutingCustomTest, AutoCommitTest, TestRecovery, DistributedSpellCheckComponentTest, ZkControllerTest, TestStressReorder, TestStressVersions, DistributedTermsComponentTest, TestSolr4Spatial, StatsComponentTest, TestGroupingSearch, SolrCmdDistributorTest, PeerSyncTest, DistributedQueryElevationComponentTest, SolrRequestParserTest, DocValuesTest, TestPseudoReturnFields, TestAtomicUpdateErrorCases, TestWordDelimiterFilterFactory, StatelessScriptUpdateProcessorFactoryTest, TestSolrDeletionPolicy1, IndexBasedSpellCheckerTest, LukeRequestHandlerTest, TestQueryTypes, PathHierarchyTokenizerFactoryTest, MoreLikeThisHandlerTest, FastVectorHighlighterTest, IndexSchemaTest, TestCSVResponseWriter, QueryParsingTest, PingRequestHandlerTest, SOLR749Test, TestQuerySenderListener, UpdateParamsTest, TestQuerySenderNoQuery, ResponseLogComponentTest, SolrIndexConfigTest, TestSolrDeletionPolicy2, SampleTest, TestSearchPerf, TestConfig, TestFuzzyAnalyzedSuggestions, TestPostingsSolrHighlighter, SynonymTokenizerTest, TestSweetSpotSimilarityFactory, TestLMDirichletSimilarityFactory, TestLMJelinekMercerSimilarityFactory, TimeZoneUtilsTest, ScriptEngineTest, PluginInfoTest, PreAnalyzedFieldTest, SpellingQueryConverterTest, ClusterStateTest, TestDocumentBuilder, SystemInfoHandlerTest, CircularListTest, DistributedMLTComponentTest, DistributedIntervalFacetingTest, AsyncCallRequestStatusResponseTest, BaseCdcrDistributedZkTest, CdcrReplicationHandlerTest, CdcrVersionReplicationTest, ChaosMonkeySafeLeaderWithPullReplicasTest, CleanupOldIndexTest, CloudExitableDirectoryReaderTest, CollectionStateFormat2Test, CreateCollectionCleanupTest, DistribDocExpirationUpdateProcessorTest, MigrateRouteKeyTest, MultiThreadedOCPTest, RecoveryAfterSoftCommitTest, SaslZkACLProviderTest, ShardSplitTest, TestOnReconnectListenerSupport, TestRandomRequestDistribution, TestRebalanceLeaders, TestReplicaProperties, TestSSLRandomization, TestShortCircuitedRequests, TestStressInPlaceUpdates, CachingDirectoryFactoryTest, TestBackupRepositoryFactory, TestConfigSets, TestInfoStreamLogging, TestSolrXml, TestSolrCloudSnapshots]
   [junit4] Completed [593/729 (3!)] on J1 in 34.77s, 1 test, 1 error <<< FAILURES!

[...truncated 43784 lines...]

[JENKINS] Lucene-Solr-7.x-MacOSX (64bit/jdk1.8.0) - Build # 123 - Failure!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-MacOSX/123/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseParallelGC

9 tests failed.
FAILED:  org.apache.solr.cloud.CollectionStateFormat2Test.testZkNodeLocation

Error Message:
Collection not created null Live Nodes: [127.0.0.1:54983_solr, 127.0.0.1:54980_solr, 127.0.0.1:54981_solr, 127.0.0.1:54982_solr] Last available state: null

Stack Trace:
java.lang.AssertionError: Collection not created
null
Live Nodes: [127.0.0.1:54983_solr, 127.0.0.1:54980_solr, 127.0.0.1:54981_solr, 127.0.0.1:54982_solr]
Last available state: null
	at __randomizedtesting.SeedInfo.seed([16871704FB707C74:592A9DA4561FF280]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:269)
	at org.apache.solr.cloud.CollectionStateFormat2Test.testZkNodeLocation(CollectionStateFormat2Test.java:50)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.CollectionStateFormat2Test

Error Message:
21 threads leaked from SUITE scope at org.apache.solr.cloud.CollectionStateFormat2Test:     1) Thread[id=11228, name=zkCallback-1725-thread-6, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=11223, name=OverseerCollectionConfigSetProcessor-98514425847545865-127.0.0.1:54981_solr-n_0000000001, state=TIMED_WAITING, group=Overseer collection creation process.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)         at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:387)         at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:193)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=11222, name=OverseerStateUpdate-98514425847545865-127.0.0.1:54981_solr-n_0000000001, state=TIMED_WAITING, group=Overseer state updater.]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:165)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:139)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:184)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=11229, name=zkCallback-1724-thread-4, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=11139, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=11225, name=zkCallback-1725-thread-3, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    7) Thread[id=11209, name=zkCallback-1724-thread-2, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    8) Thread[id=11224, name=OverseerHdfsCoreFailoverThread-98514425847545865-127.0.0.1:54981_solr-n_0000000001, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:139)         at java.lang.Thread.run(Thread.java:748)    9) Thread[id=11227, name=zkCallback-1725-thread-5, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   10) Thread[id=11231, name=zkCallback-1724-thread-6, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   11) Thread[id=11221, name=zkCallback-1724-thread-3, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   12) Thread[id=11156, name=jetty-launcher-1707-thread-2-EventThread, state=WAITING, group=TGRP-CollectionStateFormat2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)   13) Thread[id=11226, name=zkCallback-1725-thread-4, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   14) Thread[id=11207, name=zkCallback-1725-thread-2, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   15) Thread[id=11230, name=zkCallback-1724-thread-5, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   16) Thread[id=11138, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)   17) Thread[id=11157, name=jetty-launcher-1707-thread-3-SendThread(127.0.0.1:54978), state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)   18) Thread[id=11158, name=jetty-launcher-1707-thread-3-EventThread, state=WAITING, group=TGRP-CollectionStateFormat2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)   19) Thread[id=11144, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)   20) Thread[id=11141, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)   21) Thread[id=11155, name=jetty-launcher-1707-thread-2-SendThread(127.0.0.1:54978), state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 21 threads leaked from SUITE scope at org.apache.solr.cloud.CollectionStateFormat2Test: 
   1) Thread[id=11228, name=zkCallback-1725-thread-6, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=11223, name=OverseerCollectionConfigSetProcessor-98514425847545865-127.0.0.1:54981_solr-n_0000000001, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)
        at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:387)
        at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:193)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=11222, name=OverseerStateUpdate-98514425847545865-127.0.0.1:54981_solr-n_0000000001, state=TIMED_WAITING, group=Overseer state updater.]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:165)
        at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:139)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:184)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=11229, name=zkCallback-1724-thread-4, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=11139, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=11225, name=zkCallback-1725-thread-3, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   7) Thread[id=11209, name=zkCallback-1724-thread-2, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   8) Thread[id=11224, name=OverseerHdfsCoreFailoverThread-98514425847545865-127.0.0.1:54981_solr-n_0000000001, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:139)
        at java.lang.Thread.run(Thread.java:748)
   9) Thread[id=11227, name=zkCallback-1725-thread-5, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  10) Thread[id=11231, name=zkCallback-1724-thread-6, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  11) Thread[id=11221, name=zkCallback-1724-thread-3, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  12) Thread[id=11156, name=jetty-launcher-1707-thread-2-EventThread, state=WAITING, group=TGRP-CollectionStateFormat2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
  13) Thread[id=11226, name=zkCallback-1725-thread-4, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  14) Thread[id=11207, name=zkCallback-1725-thread-2, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  15) Thread[id=11230, name=zkCallback-1724-thread-5, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  16) Thread[id=11138, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
  17) Thread[id=11157, name=jetty-launcher-1707-thread-3-SendThread(127.0.0.1:54978), state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
  18) Thread[id=11158, name=jetty-launcher-1707-thread-3-EventThread, state=WAITING, group=TGRP-CollectionStateFormat2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
  19) Thread[id=11144, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
  20) Thread[id=11141, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
  21) Thread[id=11155, name=jetty-launcher-1707-thread-2-SendThread(127.0.0.1:54978), state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
	at __randomizedtesting.SeedInfo.seed([16871704FB707C74]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.CollectionStateFormat2Test

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=11228, name=zkCallback-1725-thread-6, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=11226, name=zkCallback-1725-thread-4, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=11207, name=zkCallback-1725-thread-2, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=11229, name=zkCallback-1724-thread-4, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=11230, name=zkCallback-1724-thread-5, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=11225, name=zkCallback-1725-thread-3, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    7) Thread[id=11157, name=jetty-launcher-1707-thread-3-SendThread(127.0.0.1:54978), state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)    8) Thread[id=11209, name=zkCallback-1724-thread-2, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    9) Thread[id=11227, name=zkCallback-1725-thread-5, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   10) Thread[id=11155, name=jetty-launcher-1707-thread-2-SendThread(127.0.0.1:54978), state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)   11) Thread[id=11231, name=zkCallback-1724-thread-6, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   12) Thread[id=11221, name=zkCallback-1724-thread-3, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=11228, name=zkCallback-1725-thread-6, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=11226, name=zkCallback-1725-thread-4, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=11207, name=zkCallback-1725-thread-2, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=11229, name=zkCallback-1724-thread-4, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=11230, name=zkCallback-1724-thread-5, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=11225, name=zkCallback-1725-thread-3, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   7) Thread[id=11157, name=jetty-launcher-1707-thread-3-SendThread(127.0.0.1:54978), state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
   8) Thread[id=11209, name=zkCallback-1724-thread-2, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   9) Thread[id=11227, name=zkCallback-1725-thread-5, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  10) Thread[id=11155, name=jetty-launcher-1707-thread-2-SendThread(127.0.0.1:54978), state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
  11) Thread[id=11231, name=zkCallback-1724-thread-6, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  12) Thread[id=11221, name=zkCallback-1724-thread-3, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([16871704FB707C74]:0)


FAILED:  org.apache.solr.cloud.TestPullReplica.testCreateDelete { seed=[16871704FB707C74:B9C15AB439424F68]}

Error Message:
Could not find collection : pull_replica_test_create_delete

Stack Trace:
org.apache.solr.common.SolrException: Could not find collection : pull_replica_test_create_delete
	at __randomizedtesting.SeedInfo.seed([16871704FB707C74:B9C15AB439424F68]:0)
	at org.apache.solr.common.cloud.ClusterState.getCollection(ClusterState.java:109)
	at org.apache.solr.cloud.SolrCloudTestCase.getCollectionState(SolrCloudTestCase.java:247)
	at org.apache.solr.cloud.TestPullReplica.testCreateDelete(TestPullReplica.java:161)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.TestPullReplica.testAddDocs

Error Message:
Expected collection to be created with 1 shard and 4 replicas null Live Nodes: [127.0.0.1:53957_solr, 127.0.0.1:53958_solr] Last available state: null

Stack Trace:
java.lang.AssertionError: Expected collection to be created with 1 shard and 4 replicas
null
Live Nodes: [127.0.0.1:53957_solr, 127.0.0.1:53958_solr]
Last available state: null
	at __randomizedtesting.SeedInfo.seed([16871704FB707C74:F60C0C02183E8E99]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:269)
	at org.apache.solr.cloud.TestPullReplica.testAddDocs(TestPullReplica.java:220)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.TestPullReplica.testRemoveAllWriterReplicas

Error Message:
IOException occured when talking to server at: http://127.0.0.1:53958/solr/pull_replica_test_remove_all_writer_replicas_shard1_replica_n1

Stack Trace:
org.apache.solr.client.solrj.impl.CloudSolrClient$RouteException: IOException occured when talking to server at: http://127.0.0.1:53958/solr/pull_replica_test_remove_all_writer_replicas_shard1_replica_n1
	at __randomizedtesting.SeedInfo.seed([16871704FB707C74:85898B3FEF8E95DD]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:541)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:993)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:862)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:922)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:922)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:922)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:922)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:922)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:173)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:138)
	at org.apache.solr.cloud.TestPullReplica.doTestNoLeader(TestPullReplica.java:378)
	at org.apache.solr.cloud.TestPullReplica.testRemoveAllWriterReplicas(TestPullReplica.java:286)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:53958/solr/pull_replica_test_remove_all_writer_replicas_shard1_replica_n1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:641)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.lambda$directUpdate$0(CloudSolrClient.java:516)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	... 1 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:53958 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:526)
	... 10 more


FAILED:  org.apache.solr.cloud.TestPullReplica.testAddRemovePullReplica

Error Message:
Expected collection to be created with 2 shards and 1 replica each null Live Nodes: [127.0.0.1:53957_solr, 127.0.0.1:53958_solr] Last available state: null

Stack Trace:
java.lang.AssertionError: Expected collection to be created with 2 shards and 1 replica each
null
Live Nodes: [127.0.0.1:53957_solr, 127.0.0.1:53958_solr]
Last available state: null
	at __randomizedtesting.SeedInfo.seed([16871704FB707C74:65BAA011E3C76FF0]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:269)
	at org.apache.solr.cloud.TestPullReplica.testAddRemovePullReplica(TestPullReplica.java:265)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.CustomCollectionTest.testRouteFieldForHashRouter

Error Message:
Collection not found: routeFieldColl

Stack Trace:
org.apache.solr.common.SolrException: Collection not found: routeFieldColl
	at __randomizedtesting.SeedInfo.seed([16871704FB707C74:BEB189D96411972E]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.getCollectionNames(CloudSolrClient.java:1139)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:822)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
	at org.apache.solr.client.solrj.request.UpdateRequest.commit(UpdateRequest.java:233)
	at org.apache.solr.cloud.CustomCollectionTest.testRouteFieldForHashRouter(CustomCollectionTest.java:166)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.CustomCollectionTest.testCustomCollectionsAPI

Error Message:
Could not find collection : implicitcoll

Stack Trace:
org.apache.solr.common.SolrException: Could not find collection : implicitcoll
	at __randomizedtesting.SeedInfo.seed([16871704FB707C74:7C66996FC6EACA0C]:0)
	at org.apache.solr.common.cloud.ClusterState.getCollection(ClusterState.java:109)
	at org.apache.solr.cloud.SolrCloudTestCase.getCollectionState(SolrCloudTestCase.java:247)
	at org.apache.solr.cloud.CustomCollectionTest.testCustomCollectionsAPI(CustomCollectionTest.java:68)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 11417 lines...]
   [junit4] Suite: org.apache.solr.cloud.CustomCollectionTest
   [junit4]   2> 930688 INFO  (SUITE-CustomCollectionTest-seed#[16871704FB707C74]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_16871704FB707C74-001/init-core-data-001
   [junit4]   2> 930689 WARN  (SUITE-CustomCollectionTest-seed#[16871704FB707C74]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
   [junit4]   2> 930691 INFO  (SUITE-CustomCollectionTest-seed#[16871704FB707C74]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 930693 INFO  (SUITE-CustomCollectionTest-seed#[16871704FB707C74]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth
   [junit4]   2> 930695 INFO  (SUITE-CustomCollectionTest-seed#[16871704FB707C74]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 4 servers in /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_16871704FB707C74-001/tempDir-001
   [junit4]   2> 930695 INFO  (SUITE-CustomCollectionTest-seed#[16871704FB707C74]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 930695 INFO  (Thread-1281) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 930695 INFO  (Thread-1281) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 930697 ERROR (Thread-1281) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 930802 INFO  (SUITE-CustomCollectionTest-seed#[16871704FB707C74]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:52111
   [junit4]   2> 930852 INFO  (jetty-launcher-1033-thread-1) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 930852 INFO  (jetty-launcher-1033-thread-4) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 930853 INFO  (jetty-launcher-1033-thread-3) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 930853 INFO  (jetty-launcher-1033-thread-2) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 930856 INFO  (jetty-launcher-1033-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1170c69{/solr,null,AVAILABLE}
   [junit4]   2> 930856 INFO  (jetty-launcher-1033-thread-3) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3c508f2b{/solr,null,AVAILABLE}
   [junit4]   2> 930858 INFO  (jetty-launcher-1033-thread-3) [    ] o.e.j.s.AbstractConnector Started ServerConnector@37b9713c{SSL,[ssl, http/1.1]}{127.0.0.1:52114}
   [junit4]   2> 930858 INFO  (jetty-launcher-1033-thread-3) [    ] o.e.j.s.Server Started @934456ms
   [junit4]   2> 930858 INFO  (jetty-launcher-1033-thread-3) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=52114}
   [junit4]   2> 930858 INFO  (jetty-launcher-1033-thread-4) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@695e3aa2{/solr,null,AVAILABLE}
   [junit4]   2> 930858 INFO  (jetty-launcher-1033-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@5fefc201{SSL,[ssl, http/1.1]}{127.0.0.1:52113}
   [junit4]   2> 930858 ERROR (jetty-launcher-1033-thread-3) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 930858 INFO  (jetty-launcher-1033-thread-1) [    ] o.e.j.s.Server Started @934457ms
   [junit4]   2> 930858 INFO  (jetty-launcher-1033-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=52113}
   [junit4]   2> 930858 INFO  (jetty-launcher-1033-thread-4) [    ] o.e.j.s.AbstractConnector Started ServerConnector@72505516{SSL,[ssl, http/1.1]}{127.0.0.1:52115}
   [junit4]   2> 930858 ERROR (jetty-launcher-1033-thread-1) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 930858 INFO  (jetty-launcher-1033-thread-4) [    ] o.e.j.s.Server Started @934457ms
   [junit4]   2> 930858 INFO  (jetty-launcher-1033-thread-4) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=52115}
   [junit4]   2> 930858 INFO  (jetty-launcher-1033-thread-3) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.1.0
   [junit4]   2> 930858 INFO  (jetty-launcher-1033-thread-3) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 930859 INFO  (jetty-launcher-1033-thread-3) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 930859 ERROR (jetty-launcher-1033-thread-4) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 930859 INFO  (jetty-launcher-1033-thread-3) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-08-20T06:22:30.130Z
   [junit4]   2> 930859 INFO  (jetty-launcher-1033-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.1.0
   [junit4]   2> 930859 INFO  (jetty-launcher-1033-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 930859 INFO  (jetty-launcher-1033-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 930859 INFO  (jetty-launcher-1033-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-08-20T06:22:30.130Z
   [junit4]   2> 930859 INFO  (jetty-launcher-1033-thread-4) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.1.0
   [junit4]   2> 930859 INFO  (jetty-launcher-1033-thread-4) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 930859 INFO  (jetty-launcher-1033-thread-4) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 930859 INFO  (jetty-launcher-1033-thread-4) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-08-20T06:22:30.130Z
   [junit4]   2> 930866 INFO  (jetty-launcher-1033-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6a839dcc{/solr,null,AVAILABLE}
   [junit4]   2> 930867 INFO  (jetty-launcher-1033-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@34352f08{SSL,[ssl, http/1.1]}{127.0.0.1:52119}
   [junit4]   2> 930867 INFO  (jetty-launcher-1033-thread-2) [    ] o.e.j.s.Server Started @934466ms
   [junit4]   2> 930867 INFO  (jetty-launcher-1033-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=52119}
   [junit4]   2> 930867 ERROR (jetty-launcher-1033-thread-2) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 930868 INFO  (jetty-launcher-1033-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.1.0
   [junit4]   2> 930868 INFO  (jetty-launcher-1033-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 930868 INFO  (jetty-launcher-1033-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 930868 INFO  (jetty-launcher-1033-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-08-20T06:22:30.139Z
   [junit4]   2> 930868 INFO  (jetty-launcher-1033-thread-3) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 930869 INFO  (jetty-launcher-1033-thread-4) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 930871 INFO  (jetty-launcher-1033-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 930883 INFO  (jetty-launcher-1033-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 930885 INFO  (jetty-launcher-1033-thread-3) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52111/solr
   [junit4]   2> 930887 INFO  (jetty-launcher-1033-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52111/solr
   [junit4]   2> 930892 INFO  (jetty-launcher-1033-thread-4) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52111/solr
   [junit4]   2> 930897 INFO  (jetty-launcher-1033-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52111/solr
   [jun

[...truncated too long message...]

concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    4) Thread[id=11229, name=zkCallback-1724-thread-4, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    5) Thread[id=11230, name=zkCallback-1724-thread-5, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    6) Thread[id=11225, name=zkCallback-1725-thread-3, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    7) Thread[id=11157, name=jetty-launcher-1707-thread-3-SendThread(127.0.0.1:54978), state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
   [junit4]    >    8) Thread[id=11209, name=zkCallback-1724-thread-2, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    9) Thread[id=11227, name=zkCallback-1725-thread-5, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   10) Thread[id=11155, name=jetty-launcher-1707-thread-2-SendThread(127.0.0.1:54978), state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
   [junit4]    >   11) Thread[id=11231, name=zkCallback-1724-thread-6, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   12) Thread[id=11221, name=zkCallback-1724-thread-3, state=TIMED_WAITING, group=TGRP-CollectionStateFormat2Test]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([16871704FB707C74]:0)
   [junit4] Completed [368/729 (3!)] on J0 in 191.32s, 2 tests, 1 failure, 2 errors <<< FAILURES!

[...truncated 1142 lines...]
   [junit4] JVM J0: stdout was not empty, see: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/temp/junit4-J0-20170820_060655_6209008211825749553808.sysout
   [junit4] >>> JVM J0 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: Java heap space
   [junit4] Dumping heap to /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/heapdumps/java_pid2046.hprof ...
   [junit4] Heap dump file created [104208796 bytes in 0.998 secs]
   [junit4] <<< JVM J0: EOF ----

[...truncated 7500 lines...]
BUILD FAILED
/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/build.xml:810: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/build.xml:762: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid2046.hprof

Total time: 60 minutes 28 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any

[JENKINS] Lucene-Solr-7.x-MacOSX (64bit/jdk1.8.0) - Build # 122 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-MacOSX/122/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseParallelGC

12 tests failed.
FAILED:  org.apache.solr.cloud.MultiThreadedOCPTest.test

Error Message:
org/apache/solr/client/solrj/request/CollectionAdminRequest$SplitShard

Stack Trace:
java.lang.NoClassDefFoundError: org/apache/solr/client/solrj/request/CollectionAdminRequest$SplitShard
	at __randomizedtesting.SeedInfo.seed([29DBCD100A7659D9:A18FF2CAA48A3421]:0)
	at org.apache.solr.client.solrj.request.CollectionAdminRequest.splitShard(CollectionAdminRequest.java:1010)
	at org.apache.solr.cloud.MultiThreadedOCPTest.testDeduplicationOfSubmittedTasks(MultiThreadedOCPTest.java:211)
	at org.apache.solr.cloud.MultiThreadedOCPTest.test(MultiThreadedOCPTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.ClassNotFoundException: org.apache.solr.client.solrj.request.CollectionAdminRequest$SplitShard
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	... 44 more
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-solrj/classes/java/org/apache/solr/client/solrj/request/CollectionAdminRequest$SplitShard.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 50 more


FAILED:  org.apache.solr.cloud.TestConfigSetsAPI.testDeleteErrors

Error Message:
java.io.IOException: Couldn't instantiate org.apache.zookeeper.ClientCnxnSocketNIO

Stack Trace:
org.apache.solr.common.SolrException: java.io.IOException: Couldn't instantiate org.apache.zookeeper.ClientCnxnSocketNIO
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:171)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:117)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:112)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:99)
	at org.apache.solr.cloud.MiniSolrCloudCluster.waitForAllNodes(MiniSolrCloudCluster.java:268)
	at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:262)
	at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:195)
	at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:122)
	at org.apache.solr.cloud.TestConfigSetsAPI.setUp(TestConfigSetsAPI.java:115)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:941)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.io.IOException: Couldn't instantiate org.apache.zookeeper.ClientCnxnSocketNIO
	at org.apache.zookeeper.ZooKeeper.getClientCnxnSocket(ZooKeeper.java:1842)
	at org.apache.zookeeper.ZooKeeper.<init>(ZooKeeper.java:447)
	at org.apache.zookeeper.ZooKeeper.<init>(ZooKeeper.java:380)
	at org.apache.solr.common.cloud.SolrZooKeeper.<init>(SolrZooKeeper.java:43)
	at org.apache.solr.common.cloud.ZkClientConnectionStrategy.createSolrZooKeeper(ZkClientConnectionStrategy.java:105)
	at org.apache.solr.common.cloud.DefaultConnectionStrategy.connect(DefaultConnectionStrategy.java:37)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:146)
	... 46 more
Caused by: java.io.IOException: Too many open files
	at sun.nio.ch.IOUtil.makePipe(Native Method)
	at sun.nio.ch.KQueueSelectorImpl.<init>(KQueueSelectorImpl.java:84)
	at sun.nio.ch.KQueueSelectorProvider.openSelector(KQueueSelectorProvider.java:42)
	at java.nio.channels.Selector.open(Selector.java:227)
	at org.apache.zookeeper.ClientCnxnSocketNIO.<init>(ClientCnxnSocketNIO.java:43)
	at sun.reflect.GeneratedConstructorAccessor10.newInstance(Unknown Source)
	at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
	at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
	at java.lang.Class.newInstance(Class.java:442)
	at org.apache.zookeeper.ZooKeeper.getClientCnxnSocket(ZooKeeper.java:1839)
	... 52 more


FAILED:  org.apache.solr.cloud.TestConfigSetsAPI.testUpload

Error Message:
Could not get the port for ZooKeeper server

Stack Trace:
java.lang.RuntimeException: Could not get the port for ZooKeeper server
	at org.apache.solr.cloud.ZkTestServer.run(ZkTestServer.java:515)
	at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:228)
	at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:195)
	at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:122)
	at org.apache.solr.cloud.TestConfigSetsAPI.setUp(TestConfigSetsAPI.java:115)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:941)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.TestConfigSetsAPI.testCreate

Error Message:
Could not get the port for ZooKeeper server

Stack Trace:
java.lang.RuntimeException: Could not get the port for ZooKeeper server
	at org.apache.solr.cloud.ZkTestServer.run(ZkTestServer.java:515)
	at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:228)
	at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:195)
	at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:122)
	at org.apache.solr.cloud.TestConfigSetsAPI.setUp(TestConfigSetsAPI.java:115)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:941)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.TestConfigSetsAPI.testUploadErrors

Error Message:
Could not get the port for ZooKeeper server

Stack Trace:
java.lang.RuntimeException: Could not get the port for ZooKeeper server
	at org.apache.solr.cloud.ZkTestServer.run(ZkTestServer.java:515)
	at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:228)
	at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:195)
	at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:122)
	at org.apache.solr.cloud.TestConfigSetsAPI.setUp(TestConfigSetsAPI.java:115)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:941)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.TestConfigSetsAPI.testUploadWithRunExecutableListener

Error Message:
Could not get the port for ZooKeeper server

Stack Trace:
java.lang.RuntimeException: Could not get the port for ZooKeeper server
	at org.apache.solr.cloud.ZkTestServer.run(ZkTestServer.java:515)
	at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:228)
	at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:195)
	at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:122)
	at org.apache.solr.cloud.TestConfigSetsAPI.setUp(TestConfigSetsAPI.java:115)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:941)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.TestConfigSetsAPI.testList

Error Message:
Could not get the port for ZooKeeper server

Stack Trace:
java.lang.RuntimeException: Could not get the port for ZooKeeper server
	at org.apache.solr.cloud.ZkTestServer.run(ZkTestServer.java:515)
	at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:228)
	at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:195)
	at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:122)
	at org.apache.solr.cloud.TestConfigSetsAPI.setUp(TestConfigSetsAPI.java:115)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:941)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.TestConfigSetsAPI.testUserAndTestDefaultConfigsetsAreSame

Error Message:
Could not get the port for ZooKeeper server

Stack Trace:
java.lang.RuntimeException: Could not get the port for ZooKeeper server
	at org.apache.solr.cloud.ZkTestServer.run(ZkTestServer.java:515)
	at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:228)
	at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:195)
	at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:122)
	at org.apache.solr.cloud.TestConfigSetsAPI.setUp(TestConfigSetsAPI.java:115)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:941)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.TestConfigSetsAPI.testUploadWithScriptUpdateProcessor

Error Message:
Could not get the port for ZooKeeper server

Stack Trace:
java.lang.RuntimeException: Could not get the port for ZooKeeper server
	at org.apache.solr.cloud.ZkTestServer.run(ZkTestServer.java:515)
	at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:228)
	at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:195)
	at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:122)
	at org.apache.solr.cloud.TestConfigSetsAPI.setUp(TestConfigSetsAPI.java:115)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:941)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.TestConfigSetsAPI.testCreateErrors

Error Message:
Could not get the port for ZooKeeper server

Stack Trace:
java.lang.RuntimeException: Could not get the port for ZooKeeper server
	at org.apache.solr.cloud.ZkTestServer.run(ZkTestServer.java:515)
	at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:228)
	at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:195)
	at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:122)
	at org.apache.solr.cloud.TestConfigSetsAPI.setUp(TestConfigSetsAPI.java:115)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:941)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.TestConfigSetsAPI

Error Message:
20 threads leaked from SUITE scope at org.apache.solr.cloud.TestConfigSetsAPI:     1) Thread[id=17855, name=qtp1455991284-17855, state=RUNNABLE, group=TGRP-TestConfigSetsAPI]         at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)         at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)         at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=17847, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-TestConfigSetsAPI]         at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)         at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)         at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:173)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=17860, name=qtp1455991284-17860, state=TIMED_WAITING, group=TGRP-TestConfigSetsAPI]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=17861, name=qtp1455991284-17861, state=TIMED_WAITING, group=TGRP-TestConfigSetsAPI]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=17858, name=qtp1455991284-17858, state=TIMED_WAITING, group=TGRP-TestConfigSetsAPI]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=17873, name=jetty-launcher-4595-thread-1-EventThread, state=WAITING, group=TGRP-TestConfigSetsAPI]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)    7) Thread[id=17863, name=org.eclipse.jetty.server.session.HashSessionManager@213b970cTimer, state=TIMED_WAITING, group=TGRP-TestConfigSetsAPI]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    8) Thread[id=17868, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestConfigSetsAPI]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)    9) Thread[id=17875, name=OverseerStateUpdate-98512035021324291-127.0.0.1:56948_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:165)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:139)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:184)         at java.lang.Thread.run(Thread.java:748)   10) Thread[id=17850, name=ProcessThread(sid:0 cport:56946):, state=WAITING, group=TGRP-TestConfigSetsAPI]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:122)   11) Thread[id=17862, name=qtp1455991284-17862, state=TIMED_WAITING, group=TGRP-TestConfigSetsAPI]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   12) Thread[id=17848, name=SessionTracker, state=TIMED_WAITING, group=TGRP-TestConfigSetsAPI]         at java.lang.Object.wait(Native Method)         at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146)   13) Thread[id=17856, name=qtp1455991284-17856-acceptor-0@63a2ce4b-ServerConnector@2981f8ed{HTTP/1.1,[http/1.1]}{127.0.0.1:56948}, state=RUNNABLE, group=TGRP-TestConfigSetsAPI]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:748)   14) Thread[id=17857, name=qtp1455991284-17857, state=TIMED_WAITING, group=TGRP-TestConfigSetsAPI]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   15) Thread[id=17859, name=qtp1455991284-17859, state=TIMED_WAITING, group=TGRP-TestConfigSetsAPI]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   16) Thread[id=17876, name=OverseerCollectionConfigSetProcessor-98512035021324291-127.0.0.1:56948_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.apache.solr.cloud.DistributedQueue.peekElements(DistributedQueue.java:357)         at org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:235)         at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:228)         at java.lang.Thread.run(Thread.java:748)   17) Thread[id=17846, name=Thread-2455, state=WAITING, group=TGRP-TestConfigSetsAPI]         at java.lang.Object.wait(Native Method)         at java.lang.Thread.join(Thread.java:1252)         at java.lang.Thread.join(Thread.java:1326)         at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:297)         at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:309)         at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:490)   18) Thread[id=17872, name=jetty-launcher-4595-thread-1-SendThread(127.0.0.1:56946), state=RUNNABLE, group=TGRP-TestConfigSetsAPI]         at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)         at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)         at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141)   19) Thread[id=17867, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestConfigSetsAPI]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)   20) Thread[id=17877, name=OverseerHdfsCoreFailoverThread-98512035021324291-127.0.0.1:56948_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:139)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 20 threads leaked from SUITE scope at org.apache.solr.cloud.TestConfigSetsAPI: 
   1) Thread[id=17855, name=qtp1455991284-17855, state=RUNNABLE, group=TGRP-TestConfigSetsAPI]
        at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
        at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
        at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
        at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=17847, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-TestConfigSetsAPI]
        at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
        at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
        at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:173)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=17860, name=qtp1455991284-17860, state=TIMED_WAITING, group=TGRP-TestConfigSetsAPI]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=17861, name=qtp1455991284-17861, state=TIMED_WAITING, group=TGRP-TestConfigSetsAPI]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=17858, name=qtp1455991284-17858, state=TIMED_WAITING, group=TGRP-TestConfigSetsAPI]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=17873, name=jetty-launcher-4595-thread-1-EventThread, state=WAITING, group=TGRP-TestConfigSetsAPI]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
   7) Thread[id=17863, name=org.eclipse.jetty.server.session.HashSessionManager@213b970cTimer, state=TIMED_WAITING, group=TGRP-TestConfigSetsAPI]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   8) Thread[id=17868, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestConfigSetsAPI]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
   9) Thread[id=17875, name=OverseerStateUpdate-98512035021324291-127.0.0.1:56948_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:165)
        at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:139)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:184)
        at java.lang.Thread.run(Thread.java:748)
  10) Thread[id=17850, name=ProcessThread(sid:0 cport:56946):, state=WAITING, group=TGRP-TestConfigSetsAPI]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:122)
  11) Thread[id=17862, name=qtp1455991284-17862, state=TIMED_WAITING, group=TGRP-TestConfigSetsAPI]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  12) Thread[id=17848, name=SessionTracker, state=TIMED_WAITING, group=TGRP-TestConfigSetsAPI]
        at java.lang.Object.wait(Native Method)
        at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146)
  13) Thread[id=17856, name=qtp1455991284-17856-acceptor-0@63a2ce4b-ServerConnector@2981f8ed{HTTP/1.1,[http/1.1]}{127.0.0.1:56948}, state=RUNNABLE, group=TGRP-TestConfigSetsAPI]
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
        at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371)
        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:748)
  14) Thread[id=17857, name=qtp1455991284-17857, state=TIMED_WAITING, group=TGRP-TestConfigSetsAPI]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  15) Thread[id=17859, name=qtp1455991284-17859, state=TIMED_WAITING, group=TGRP-TestConfigSetsAPI]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  16) Thread[id=17876, name=OverseerCollectionConfigSetProcessor-98512035021324291-127.0.0.1:56948_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.apache.solr.cloud.DistributedQueue.peekElements(DistributedQueue.java:357)
        at org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:235)
        at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:228)
        at java.lang.Thread.run(Thread.java:748)
  17) Thread[id=17846, name=Thread-2455, state=WAITING, group=TGRP-TestConfigSetsAPI]
        at java.lang.Object.wait(Native Method)
        at java.lang.Thread.join(Thread.java:1252)
        at java.lang.Thread.join(Thread.java:1326)
        at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:297)
        at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:309)
        at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:490)
  18) Thread[id=17872, name=jetty-launcher-4595-thread-1-SendThread(127.0.0.1:56946), state=RUNNABLE, group=TGRP-TestConfigSetsAPI]
        at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
        at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
        at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141)
  19) Thread[id=17867, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestConfigSetsAPI]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
  20) Thread[id=17877, name=OverseerHdfsCoreFailoverThread-98512035021324291-127.0.0.1:56948_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:139)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([29DBCD100A7659D9]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.TestConfigSetsAPI

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=17855, name=qtp1455991284-17855, state=RUNNABLE, group=TGRP-TestConfigSetsAPI]         at sun.nio.ch.KQueueArrayWrapper.interrupt(Native Method)         at sun.nio.ch.KQueueArrayWrapper.interrupt(KQueueArrayWrapper.java:203)         at sun.nio.ch.KQueueSelectorImpl.wakeup(KQueueSelectorImpl.java:257)         at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)         at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)         at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:116)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=17872, name=jetty-launcher-4595-thread-1-SendThread(127.0.0.1:56946), state=TIMED_WAITING, group=TGRP-TestConfigSetsAPI]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)    3) Thread[id=17863, name=org.eclipse.jetty.server.session.HashSessionManager@213b970cTimer, state=TIMED_WAITING, group=TGRP-TestConfigSetsAPI]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=17855, name=qtp1455991284-17855, state=RUNNABLE, group=TGRP-TestConfigSetsAPI]
        at sun.nio.ch.KQueueArrayWrapper.interrupt(Native Method)
        at sun.nio.ch.KQueueArrayWrapper.interrupt(KQueueArrayWrapper.java:203)
        at sun.nio.ch.KQueueSelectorImpl.wakeup(KQueueSelectorImpl.java:257)
        at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
        at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
        at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:116)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
        at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=17872, name=jetty-launcher-4595-thread-1-SendThread(127.0.0.1:56946), state=TIMED_WAITING, group=TGRP-TestConfigSetsAPI]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
   3) Thread[id=17863, name=org.eclipse.jetty.server.session.HashSessionManager@213b970cTimer, state=TIMED_WAITING, group=TGRP-TestConfigSetsAPI]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([29DBCD100A7659D9]:0)




Build Log:
[...truncated 11719 lines...]
   [junit4] Suite: org.apache.solr.cloud.MultiThreadedOCPTest
   [junit4]   2> 1227225 INFO  (SUITE-MultiThreadedOCPTest-seed#[29DBCD100A7659D9]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_29DBCD100A7659D9-001/init-core-data-001
   [junit4]   2> 1227225 WARN  (SUITE-MultiThreadedOCPTest-seed#[29DBCD100A7659D9]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=3 numCloses=3
   [junit4]   2> 1227225 INFO  (SUITE-MultiThreadedOCPTest-seed#[29DBCD100A7659D9]-worker) [    ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 1227228 INFO  (SUITE-MultiThreadedOCPTest-seed#[29DBCD100A7659D9]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth
   [junit4]   2> 1227228 INFO  (SUITE-MultiThreadedOCPTest-seed#[29DBCD100A7659D9]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
   [junit4]   2> 1227229 INFO  (TEST-MultiThreadedOCPTest.test-seed#[29DBCD100A7659D9]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1227229 INFO  (Thread-2393) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1227229 INFO  (Thread-2393) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1227232 ERROR (Thread-2393) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 1227335 INFO  (TEST-MultiThreadedOCPTest.test-seed#[29DBCD100A7659D9]) [    ] o.a.s.c.ZkTestServer start zk server on port:56547
   [junit4]   2> 1227397 INFO  (TEST-MultiThreadedOCPTest.test-seed#[29DBCD100A7659D9]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1227401 INFO  (TEST-MultiThreadedOCPTest.test-seed#[29DBCD100A7659D9]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1227403 INFO  (TEST-MultiThreadedOCPTest.test-seed#[29DBCD100A7659D9]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1227405 INFO  (TEST-MultiThreadedOCPTest.test-seed#[29DBCD100A7659D9]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1227407 INFO  (TEST-MultiThreadedOCPTest.test-seed#[29DBCD100A7659D9]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1227408 INFO  (TEST-MultiThreadedOCPTest.test-seed#[29DBCD100A7659D9]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1227410 INFO  (TEST-MultiThreadedOCPTest.test-seed#[29DBCD100A7659D9]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1227412 INFO  (TEST-MultiThreadedOCPTest.test-seed#[29DBCD100A7659D9]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1227414 INFO  (TEST-MultiThreadedOCPTest.test-seed#[29DBCD100A7659D9]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1227416 INFO  (TEST-MultiThreadedOCPTest.test-seed#[29DBCD100A7659D9]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1227418 INFO  (TEST-MultiThreadedOCPTest.test-seed#[29DBCD100A7659D9]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1227421 INFO  (TEST-MultiThreadedOCPTest.test-seed#[29DBCD100A7659D9]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
   [junit4]   2> 1227421 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15dfc2c19cd0001, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1227822 INFO  (TEST-MultiThreadedOCPTest.test-seed#[29DBCD100A7659D9]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1227824 INFO  (TEST-MultiThreadedOCPTest.test-seed#[29DBCD100A7659D9]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4886d96c{/,null,AVAILABLE}
   [junit4]   2> 1227825 INFO  (TEST-MultiThreadedOCPTest.test-seed#[29DBCD100A7659D9]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@eec7068{HTTP/1.1,[http/1.1]}{127.0.0.1:56551}
   [junit4]   2> 1227826 INFO  (TEST-MultiThreadedOCPTest.test-seed#[29DBCD100A7659D9]) [    ] o.e.j.s.Server Started @1231520ms
   [junit4]   2> 1227826 INFO  (TEST-MultiThreadedOCPTest.test-seed#[29DBCD100A7659D9]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_29DBCD100A7659D9-001/tempDir-001/control/data, replicaType=NRT, hostContext=/, hostPort=56551, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_29DBCD100A7659D9-001/control-001/cores}
   [junit4]   2> 1227826 ERROR (TEST-MultiThreadedOCPTest.test-seed#[29DBCD100A7659D9]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1227826 INFO  (TEST-MultiThreadedOCPTest.test-seed#[29DBCD100A7659D9]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.1.0
   [junit4]   2> 1227826 INFO  (TEST-MultiThreadedOCPTest.test-seed#[29DBCD100A7659D9]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1227826 INFO  (TEST-MultiThreadedOCPTest.test-seed#[29DBCD100A7659D9]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 1227826 INFO  (TEST-MultiThreadedOCPTest.test-seed#[29DBCD100A7659D9]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-08-19T20:25:35.520Z
   [junit4]   2> 1227829 INFO  (TEST-MultiThreadedOCPTest.test-seed#[29DBCD100A7659D9]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 1227829 INFO  (TEST-MultiThreadedOCPTest.test-seed#[29DBCD100A7659D9]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_29DBCD100A7659D9-001/control-001/solr.xml
   [junit4]   2> 1227833 INFO  (TEST-MultiThreadedOCPTest.test-seed#[29DBCD100A7659D9]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@5a92cb18, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 1227836 INFO  (TEST-MultiThreadedOCPTest.test-seed#[29DBCD100A7659D9]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:56547/solr
   [junit4]   2> 1227959 INFO  (TEST-MultiThreadedOCPTest.test-seed#[29DBCD100A7659D9]) [n:127.0.0.1:56551_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1227960 INFO  (TEST-MultiThreadedOCPTest.test-seed#[29DBCD100A7659D9]) [n:127.0.0.1:56551_    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:56551_
   [junit4]   2> 1227961 INFO  (TEST-MultiThreadedOCPTest.test-seed#[29DBCD100A7659D9]) [n:127.0.0.1:56551_    ] o.a.s.c.Overseer Overseer (id=98512033213644804-127.0.0.1:56551_-n_0000000000) starting
   [junit4]   2> 1227970 INFO  (TEST-MultiThreadedOCPTest.test-seed#[29DBCD100A7659D9]) [n:127.0.0.1:56551_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:56551_
   [junit4]   2> 1227973 INFO  (zkCallback-4548-thread-1-processing-n:127.0.0.1:56551_) [n:127.0.0.1:56551_    ] o.a.s.c.c.ZkStateReader Updated live nodes from Zo

[...truncated too long message...]

    >   13) Thread[id=17856, name=qtp1455991284-17856-acceptor-0@63a2ce4b-ServerConnector@2981f8ed{HTTP/1.1,[http/1.1]}{127.0.0.1:56948}, state=RUNNABLE, group=TGRP-TestConfigSetsAPI]
   [junit4]    >         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
   [junit4]    >         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
   [junit4]    >         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
   [junit4]    >         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371)
   [junit4]    >         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   14) Thread[id=17857, name=qtp1455991284-17857, state=TIMED_WAITING, group=TGRP-TestConfigSetsAPI]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   15) Thread[id=17859, name=qtp1455991284-17859, state=TIMED_WAITING, group=TGRP-TestConfigSetsAPI]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   16) Thread[id=17876, name=OverseerCollectionConfigSetProcessor-98512035021324291-127.0.0.1:56948_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at org.apache.solr.cloud.DistributedQueue.peekElements(DistributedQueue.java:357)
   [junit4]    >         at org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:235)
   [junit4]    >         at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:228)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   17) Thread[id=17846, name=Thread-2455, state=WAITING, group=TGRP-TestConfigSetsAPI]
   [junit4]    >         at java.lang.Object.wait(Native Method)
   [junit4]    >         at java.lang.Thread.join(Thread.java:1252)
   [junit4]    >         at java.lang.Thread.join(Thread.java:1326)
   [junit4]    >         at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:297)
   [junit4]    >         at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:309)
   [junit4]    >         at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:490)
   [junit4]    >   18) Thread[id=17872, name=jetty-launcher-4595-thread-1-SendThread(127.0.0.1:56946), state=RUNNABLE, group=TGRP-TestConfigSetsAPI]
   [junit4]    >         at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
   [junit4]    >         at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
   [junit4]    >         at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)
   [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]    >         at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141)
   [junit4]    >   19) Thread[id=17867, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestConfigSetsAPI]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   20) Thread[id=17877, name=OverseerHdfsCoreFailoverThread-98512035021324291-127.0.0.1:56948_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:139)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([29DBCD100A7659D9]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   [junit4]    >    1) Thread[id=17855, name=qtp1455991284-17855, state=RUNNABLE, group=TGRP-TestConfigSetsAPI]
   [junit4]    >         at sun.nio.ch.KQueueArrayWrapper.interrupt(Native Method)
   [junit4]    >         at sun.nio.ch.KQueueArrayWrapper.interrupt(KQueueArrayWrapper.java:203)
   [junit4]    >         at sun.nio.ch.KQueueSelectorImpl.wakeup(KQueueSelectorImpl.java:257)
   [junit4]    >         at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
   [junit4]    >         at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
   [junit4]    >         at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:116)
   [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    2) Thread[id=17872, name=jetty-launcher-4595-thread-1-SendThread(127.0.0.1:56946), state=TIMED_WAITING, group=TGRP-TestConfigSetsAPI]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
   [junit4]    >    3) Thread[id=17863, name=org.eclipse.jetty.server.session.HashSessionManager@213b970cTimer, state=TIMED_WAITING, group=TGRP-TestConfigSetsAPI]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([29DBCD100A7659D9]:0)
   [junit4] Completed [309/729 (2!)] on J1 in 456.93s, 10 tests, 11 errors <<< FAILURES!

[...truncated 44683 lines...]

[JENKINS] Lucene-Solr-7.x-MacOSX (64bit/jdk1.8.0) - Build # 121 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-MacOSX/121/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseSerialGC

86 tests failed.
FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
Doc with id=1 not found in http://127.0.0.1:52720/cdk/j/collMinRf_1x3 due to: Path not found: /id; rsp={doc=null}

Stack Trace:
java.lang.AssertionError: Doc with id=1 not found in http://127.0.0.1:52720/cdk/j/collMinRf_1x3 due to: Path not found: /id; rsp={doc=null}
	at __randomizedtesting.SeedInfo.seed([A4AB1C166DFC83B8:2CFF23CCC300EE40]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.cloud.HttpPartitionTest.assertDocExists(HttpPartitionTest.java:603)
	at org.apache.solr.cloud.HttpPartitionTest.assertDocsExistInAllReplicas(HttpPartitionTest.java:558)
	at org.apache.solr.cloud.HttpPartitionTest.testMinRf(HttpPartitionTest.java:249)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:127)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.SolrExampleBinaryTest

Error Message:
org.apache.solr.client.solrj.SolrExampleBinaryTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.client.solrj.SolrExampleBinaryTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-solrj/classes/test/org/apache/solr/client/solrj/SolrExampleBinaryTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.SolrExampleXMLTest

Error Message:
org.apache.solr.client.solrj.SolrExampleXMLTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.client.solrj.SolrExampleXMLTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-solrj/classes/test/org/apache/solr/client/solrj/SolrExampleXMLTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.SolrSchemalessExampleTest

Error Message:
org.apache.solr.client.solrj.SolrSchemalessExampleTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.client.solrj.SolrSchemalessExampleTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-solrj/classes/test/org/apache/solr/client/solrj/SolrSchemalessExampleTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.TestBatchUpdate

Error Message:
org.apache.solr.client.solrj.TestBatchUpdate

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.client.solrj.TestBatchUpdate
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-solrj/classes/test/org/apache/solr/client/solrj/TestBatchUpdate.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.beans.TestDocumentObjectBinder

Error Message:
org.apache.solr.client.solrj.beans.TestDocumentObjectBinder

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.client.solrj.beans.TestDocumentObjectBinder
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-solrj/classes/test/org/apache/solr/client/solrj/beans/TestDocumentObjectBinder.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.embedded.JettyWebappTest

Error Message:
org.apache.solr.client.solrj.embedded.JettyWebappTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.client.solrj.embedded.JettyWebappTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-solrj/classes/test/org/apache/solr/client/solrj/embedded/JettyWebappTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.embedded.MergeIndexesEmbeddedTest

Error Message:
org.apache.solr.client.solrj.embedded.MergeIndexesEmbeddedTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.client.solrj.embedded.MergeIndexesEmbeddedTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-solrj/classes/test/org/apache/solr/client/solrj/embedded/MergeIndexesEmbeddedTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.embedded.SolrExampleEmbeddedTest

Error Message:
org.apache.solr.client.solrj.embedded.SolrExampleEmbeddedTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.client.solrj.embedded.SolrExampleEmbeddedTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-solrj/classes/test/org/apache/solr/client/solrj/embedded/SolrExampleEmbeddedTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.embedded.SolrExampleStreamingBinaryTest

Error Message:
org.apache.solr.client.solrj.embedded.SolrExampleStreamingBinaryTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.client.solrj.embedded.SolrExampleStreamingBinaryTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-solrj/classes/test/org/apache/solr/client/solrj/embedded/SolrExampleStreamingBinaryTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.embedded.SolrExampleStreamingTest

Error Message:
org.apache.solr.client.solrj.embedded.SolrExampleStreamingTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.client.solrj.embedded.SolrExampleStreamingTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-solrj/classes/test/org/apache/solr/client/solrj/embedded/SolrExampleStreamingTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.embedded.TestSolrProperties

Error Message:
org.apache.solr.client.solrj.embedded.TestSolrProperties

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.client.solrj.embedded.TestSolrProperties
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-solrj/classes/test/org/apache/solr/client/solrj/embedded/TestSolrProperties.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.impl.CloudSolrClientCacheTest

Error Message:
org.apache.solr.client.solrj.impl.CloudSolrClientCacheTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.client.solrj.impl.CloudSolrClientCacheTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-solrj/classes/test/org/apache/solr/client/solrj/impl/CloudSolrClientCacheTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.impl.CloudSolrClientTest

Error Message:
org.apache.solr.client.solrj.impl.CloudSolrClientTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.client.solrj.impl.CloudSolrClientTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-solrj/classes/test/org/apache/solr/client/solrj/impl/CloudSolrClientTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClientBuilderTest

Error Message:
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClientBuilderTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClientBuilderTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-solrj/classes/test/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClientBuilderTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClientTest

Error Message:
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClientTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClientTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-solrj/classes/test/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClientTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.impl.HttpSolrClientBuilderTest

Error Message:
org.apache.solr.client.solrj.impl.HttpSolrClientBuilderTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.client.solrj.impl.HttpSolrClientBuilderTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-solrj/classes/test/org/apache/solr/client/solrj/impl/HttpSolrClientBuilderTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.impl.HttpSolrClientConPoolTest

Error Message:
org.apache.solr.client.solrj.impl.HttpSolrClientConPoolTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.client.solrj.impl.HttpSolrClientConPoolTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-solrj/classes/test/org/apache/solr/client/solrj/impl/HttpSolrClientConPoolTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.impl.HttpSolrClientSSLAuthConPoolTest

Error Message:
org.apache.solr.client.solrj.impl.HttpSolrClientSSLAuthConPoolTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.client.solrj.impl.HttpSolrClientSSLAuthConPoolTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-solrj/classes/test/org/apache/solr/client/solrj/impl/HttpSolrClientSSLAuthConPoolTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.impl.LBHttpSolrClientBuilderTest

Error Message:
org.apache.solr.client.solrj.impl.LBHttpSolrClientBuilderTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.client.solrj.impl.LBHttpSolrClientBuilderTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-solrj/classes/test/org/apache/solr/client/solrj/impl/LBHttpSolrClientBuilderTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.impl.SolrPortAwareCookieSpecTest

Error Message:
org.apache.solr.client.solrj.impl.SolrPortAwareCookieSpecTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.client.solrj.impl.SolrPortAwareCookieSpecTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-solrj/classes/test/org/apache/solr/client/solrj/impl/SolrPortAwareCookieSpecTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.impl.TestCloudSolrClientConnections

Error Message:
org.apache.solr.client.solrj.impl.TestCloudSolrClientConnections

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.client.solrj.impl.TestCloudSolrClientConnections
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-solrj/classes/test/org/apache/solr/client/solrj/impl/TestCloudSolrClientConnections.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.io.graph.GraphExpressionTest

Error Message:
org.apache.solr.client.solrj.io.graph.GraphExpressionTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.client.solrj.io.graph.GraphExpressionTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-solrj/classes/test/org/apache/solr/client/solrj/io/graph/GraphExpressionTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.io.graph.GraphTest

Error Message:
org.apache.solr.client.solrj.io.graph.GraphTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.client.solrj.io.graph.GraphTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-solrj/classes/test/org/apache/solr/client/solrj/io/graph/GraphTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.io.sql.JdbcDriverTest

Error Message:
org.apache.solr.client.solrj.io.sql.JdbcDriverTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.client.solrj.io.sql.JdbcDriverTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-solrj/classes/test/org/apache/solr/client/solrj/io/sql/JdbcDriverTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.io.stream.JDBCStreamTest

Error Message:
org.apache.solr.client.solrj.io.stream.JDBCStreamTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.client.solrj.io.stream.JDBCStreamTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-solrj/classes/test/org/apache/solr/client/solrj/io/stream/JDBCStreamTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.io.stream.SelectWithEvaluatorsTest

Error Message:
org.apache.solr.client.solrj.io.stream.SelectWithEvaluatorsTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.client.solrj.io.stream.SelectWithEvaluatorsTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-solrj/classes/test/org/apache/solr/client/solrj/io/stream/SelectWithEvaluatorsTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.io.stream.StreamExpressionTest

Error Message:
org.apache.solr.client.solrj.io.stream.StreamExpressionTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.client.solrj.io.stream.StreamExpressionTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-solrj/classes/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.io.stream.StreamExpressionToExpessionTest

Error Message:
org.apache.solr.client.solrj.io.stream.StreamExpressionToExpessionTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.client.solrj.io.stream.StreamExpressionToExpessionTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-solrj/classes/test/org/apache/solr/client/solrj/io/stream/StreamExpressionToExpessionTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.io.stream.StreamExpressionToExplanationTest

Error Message:
org.apache.solr.client.solrj.io.stream.StreamExpressionToExplanationTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.client.solrj.io.stream.StreamExpressionToExplanationTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-solrj/classes/test/org/apache/solr/client/solrj/io/stream/StreamExpressionToExplanationTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.io.stream.eval.AbsoluteValueEvaluatorTest

Error Message:
org.apache.solr.client.solrj.io.stream.eval.AbsoluteValueEvaluatorTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.client.solrj.io.stream.eval.AbsoluteValueEvaluatorTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-solrj/classes/test/org/apache/solr/client/solrj/io/stream/eval/AbsoluteValueEvaluatorTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.io.stream.eval.AddEvaluatorTest

Error Message:
org.apache.solr.client.solrj.io.stream.eval.AddEvaluatorTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.client.solrj.io.stream.eval.AddEvaluatorTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-solrj/classes/test/org/apache/solr/client/solrj/io/stream/eval/AddEvaluatorTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.io.stream.eval.ArcCosineEvaluatorTest

Error Message:
org.apache.solr.client.solrj.io.stream.eval.ArcCosineEvaluatorTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.client.solrj.io.stream.eval.ArcCosineEvaluatorTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-solrj/classes/test/org/apache/solr/client/solrj/io/stream/eval/ArcCosineEvaluatorTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.io.stream.eval.ArcSineEvaluatorTest

Error Message:
org.apache.solr.client.solrj.io.stream.eval.ArcSineEvaluatorTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.client.solrj.io.stream.eval.ArcSineEvaluatorTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-solrj/classes/test/org/apache/solr/client/solrj/io/stream/eval/ArcSineEvaluatorTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.io.stream.eval.ArcTangentEvaluatorTest

Error Message:
org.apache.solr.client.solrj.io.stream.eval.ArcTangentEvaluatorTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.client.solrj.io.stream.eval.ArcTangentEvaluatorTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-solrj/classes/test/org/apache/solr/client/solrj/io/stream/eval/ArcTangentEvaluatorTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.io.stream.eval.ArrayEvaluatorTest

Error Message:
org.apache.solr.client.solrj.io.stream.eval.ArrayEvaluatorTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.client.solrj.io.stream.eval.ArrayEvaluatorTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-solrj/classes/test/org/apache/solr/client/solrj/io/stream/eval/ArrayEvaluatorTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.io.stream.eval.ConversionEvaluatorsTest

Error Message:
org.apache.solr.client.solrj.io.stream.eval.ConversionEvaluatorsTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.client.solrj.io.stream.eval.ConversionEvaluatorsTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-solrj/classes/test/org/apache/solr/client/solrj/io/stream/eval/ConversionEvaluatorsTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.io.stream.eval.CosineEvaluatorTest

Error Message:
org.apache.solr.client.solrj.io.stream.eval.CosineEvaluatorTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.client.solrj.io.stream.eval.CosineEvaluatorTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-solrj/classes/test/org/apache/solr/client/solrj/io/stream/eval/CosineEvaluatorTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.io.stream.eval.DivideEvaluatorTest

Error Message:
org.apache.solr.client.solrj.io.stream.eval.DivideEvaluatorTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.client.solrj.io.stream.eval.DivideEvaluatorTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-solrj/classes/test/org/apache/solr/client/solrj/io/stream/eval/DivideEvaluatorTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.io.stream.eval.ExclusiveOrEvaluatorTest

Error Message:
org.apache.solr.client.solrj.io.stream.eval.ExclusiveOrEvaluatorTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.client.solrj.io.stream.eval.ExclusiveOrEvaluatorTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-solrj/classes/test/org/apache/solr/client/solrj/io/stream/eval/ExclusiveOrEvaluatorTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.io.stream.eval.FieldEvaluatorTest

Error Message:
org.apache.solr.client.solrj.io.stream.eval.FieldEvaluatorTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.client.solrj.io.stream.eval.FieldEvaluatorTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-solrj/classes/test/org/apache/solr/client/solrj/io/stream/eval/FieldEvaluatorTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.io.stream.eval.FloorEvaluatorTest

Error Message:
org.apache.solr.client.solrj.io.stream.eval.FloorEvaluatorTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.client.solrj.io.stream.eval.FloorEvaluatorTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-solrj/classes/test/org/apache/solr/client/solrj/io/stream/eval/FloorEvaluatorTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.io.stream.eval.GreaterThanEqualToEvaluatorTest

Error Message:
org.apache.solr.client.solrj.io.stream.eval.GreaterThanEqualToEvaluatorTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.client.solrj.io.stream.eval.GreaterThanEqualToEvaluatorTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-solrj/classes/test/org/apache/solr/client/solrj/io/stream/eval/GreaterThanEqualToEvaluatorTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.io.stream.eval.GreaterThanEvaluatorTest

Error Message:
org.apache.solr.client.solrj.io.stream.eval.GreaterThanEvaluatorTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.client.solrj.io.stream.eval.GreaterThanEvaluatorTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-solrj/classes/test/org/apache/solr/client/solrj/io/stream/eval/GreaterThanEvaluatorTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.io.stream.eval.HyperbolicCosineEvaluatorTest

Error Message:
org.apache.solr.client.solrj.io.stream.eval.HyperbolicCosineEvaluatorTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.client.solrj.io.stream.eval.HyperbolicCosineEvaluatorTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-solrj/classes/test/org/apache/solr/client/solrj/io/stream/eval/HyperbolicCosineEvaluatorTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.io.stream.eval.HyperbolicSineEvaluatorTest

Error Message:
org.apache.solr.client.solrj.io.stream.eval.HyperbolicSineEvaluatorTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.client.solrj.io.stream.eval.HyperbolicSineEvaluatorTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at c

[...truncated too long message...]

nit4] Completed [111/135 (78!)] on J0 in 0.00s, 0 tests, 1 error <<< FAILURES!

[...truncated 19 lines...]
   [junit4] Suite: org.apache.solr.common.cloud.TestCollectionStateWatchers
   [junit4] ERROR   0.00s J0 | TestCollectionStateWatchers (suite) <<<
   [junit4]    > Throwable #1: java.lang.ClassNotFoundException: org.apache.solr.common.cloud.TestCollectionStateWatchers
   [junit4]    > 	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
   [junit4]    > 	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
   [junit4]    > 	at java.security.AccessController.doPrivileged(Native Method)
   [junit4]    > 	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
   [junit4]    > 	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
   [junit4]    > 	at java.lang.Class.forName0(Native Method)
   [junit4]    > 	at java.lang.Class.forName(Class.java:348)
   [junit4]    > Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-solrj/classes/test/org/apache/solr/common/cloud/TestCollectionStateWatchers.class (Too many open files)
   [junit4]    > 	at java.io.FileInputStream.open0(Native Method)
   [junit4]    > 	at java.io.FileInputStream.open(FileInputStream.java:195)
   [junit4]    > 	at java.io.FileInputStream.<init>(FileInputStream.java:138)
   [junit4]    > 	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
   [junit4]    > 	at sun.misc.Resource.cachedInputStream(Resource.java:77)
   [junit4]    > 	at sun.misc.Resource.getByteBuffer(Resource.java:160)
   [junit4]    > 	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
   [junit4]    > 	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
   [junit4]    > 	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
   [junit4]    > 	... 12 more
   [junit4] Completed [118/135 (79!)] on J0 in 0.00s, 0 tests, 1 error <<< FAILURES!

[...truncated 1 lines...]
   [junit4] Suite: org.apache.solr.common.params.CommonAdminParamsTest
   [junit4] ERROR   0.00s J0 | CommonAdminParamsTest (suite) <<<
   [junit4]    > Throwable #1: java.lang.ClassNotFoundException: org.apache.solr.common.params.CommonAdminParamsTest
   [junit4]    > 	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
   [junit4]    > 	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
   [junit4]    > 	at java.security.AccessController.doPrivileged(Native Method)
   [junit4]    > 	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
   [junit4]    > 	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
   [junit4]    > 	at java.lang.Class.forName0(Native Method)
   [junit4]    > 	at java.lang.Class.forName(Class.java:348)
   [junit4]    > Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-solrj/classes/test/org/apache/solr/common/params/CommonAdminParamsTest.class (Too many open files)
   [junit4]    > 	at java.io.FileInputStream.open0(Native Method)
   [junit4]    > 	at java.io.FileInputStream.open(FileInputStream.java:195)
   [junit4]    > 	at java.io.FileInputStream.<init>(FileInputStream.java:138)
   [junit4]    > 	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
   [junit4]    > 	at sun.misc.Resource.cachedInputStream(Resource.java:77)
   [junit4]    > 	at sun.misc.Resource.getByteBuffer(Resource.java:160)
   [junit4]    > 	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
   [junit4]    > 	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
   [junit4]    > 	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
   [junit4]    > 	... 12 more
   [junit4] Completed [119/135 (80!)] on J0 in 0.00s, 0 tests, 1 error <<< FAILURES!

[...truncated 13 lines...]
   [junit4] Suite: org.apache.solr.common.params.CommonParamsTest
   [junit4] ERROR   0.00s J0 | CommonParamsTest (suite) <<<
   [junit4]    > Throwable #1: java.lang.ClassNotFoundException: org.apache.solr.common.params.CommonParamsTest
   [junit4]    > 	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
   [junit4]    > 	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
   [junit4]    > 	at java.security.AccessController.doPrivileged(Native Method)
   [junit4]    > 	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
   [junit4]    > 	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
   [junit4]    > 	at java.lang.Class.forName0(Native Method)
   [junit4]    > 	at java.lang.Class.forName(Class.java:348)
   [junit4]    > Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-solrj/classes/test/org/apache/solr/common/params/CommonParamsTest.class (Too many open files)
   [junit4]    > 	at java.io.FileInputStream.open0(Native Method)
   [junit4]    > 	at java.io.FileInputStream.open(FileInputStream.java:195)
   [junit4]    > 	at java.io.FileInputStream.<init>(FileInputStream.java:138)
   [junit4]    > 	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
   [junit4]    > 	at sun.misc.Resource.cachedInputStream(Resource.java:77)
   [junit4]    > 	at sun.misc.Resource.getByteBuffer(Resource.java:160)
   [junit4]    > 	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
   [junit4]    > 	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
   [junit4]    > 	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
   [junit4]    > 	... 12 more
   [junit4] Completed [124/135 (81!)] on J0 in 0.00s, 0 tests, 1 error <<< FAILURES!

[...truncated 1 lines...]
   [junit4] Suite: org.apache.solr.common.util.JsonValidatorTest
   [junit4] ERROR   0.00s J0 | JsonValidatorTest (suite) <<<
   [junit4]    > Throwable #1: java.lang.ClassNotFoundException: org.apache.solr.common.util.JsonValidatorTest
   [junit4]    > 	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
   [junit4]    > 	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
   [junit4]    > 	at java.security.AccessController.doPrivileged(Native Method)
   [junit4]    > 	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
   [junit4]    > 	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
   [junit4]    > 	at java.lang.Class.forName0(Native Method)
   [junit4]    > 	at java.lang.Class.forName(Class.java:348)
   [junit4]    > Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-solrj/classes/test/org/apache/solr/common/util/JsonValidatorTest.class (Too many open files)
   [junit4]    > 	at java.io.FileInputStream.open0(Native Method)
   [junit4]    > 	at java.io.FileInputStream.open(FileInputStream.java:195)
   [junit4]    > 	at java.io.FileInputStream.<init>(FileInputStream.java:138)
   [junit4]    > 	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
   [junit4]    > 	at sun.misc.Resource.cachedInputStream(Resource.java:77)
   [junit4]    > 	at sun.misc.Resource.getByteBuffer(Resource.java:160)
   [junit4]    > 	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
   [junit4]    > 	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
   [junit4]    > 	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
   [junit4]    > 	... 12 more
   [junit4] Completed [125/135 (82!)] on J0 in 0.00s, 0 tests, 1 error <<< FAILURES!

[...truncated 4 lines...]
   [junit4] Suite: org.apache.solr.common.util.TestJsonRecordReader
   [junit4] ERROR   0.00s J0 | TestJsonRecordReader (suite) <<<
   [junit4]    > Throwable #1: java.lang.ClassNotFoundException: org.apache.solr.common.util.TestJsonRecordReader
   [junit4]    > 	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
   [junit4]    > 	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
   [junit4]    > 	at java.security.AccessController.doPrivileged(Native Method)
   [junit4]    > 	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
   [junit4]    > 	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
   [junit4]    > 	at java.lang.Class.forName0(Native Method)
   [junit4]    > 	at java.lang.Class.forName(Class.java:348)
   [junit4]    > Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-solrj/classes/test/org/apache/solr/common/util/TestJsonRecordReader.class (Too many open files)
   [junit4]    > 	at java.io.FileInputStream.open0(Native Method)
   [junit4]    > 	at java.io.FileInputStream.open(FileInputStream.java:195)
   [junit4]    > 	at java.io.FileInputStream.<init>(FileInputStream.java:138)
   [junit4]    > 	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
   [junit4]    > 	at sun.misc.Resource.cachedInputStream(Resource.java:77)
   [junit4]    > 	at sun.misc.Resource.getByteBuffer(Resource.java:160)
   [junit4]    > 	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
   [junit4]    > 	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
   [junit4]    > 	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
   [junit4]    > 	... 12 more
   [junit4] Completed [127/135 (83!)] on J0 in 0.00s, 0 tests, 1 error <<< FAILURES!

[...truncated 42557 lines...]

[JENKINS] Lucene-Solr-7.x-MacOSX (64bit/jdk1.8.0) - Build # 120 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-MacOSX/120/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseSerialGC

1 tests failed.
FAILED:  org.apache.solr.cloud.DocValuesNotIndexedTest.testGroupingDVOnly

Error Message:
Unexpected number of elements in the group for intGSF: 8

Stack Trace:
java.lang.AssertionError: Unexpected number of elements in the group for intGSF: 8
	at __randomizedtesting.SeedInfo.seed([B6E7B93B17A8657A:2D5CD7635AF05724]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.DocValuesNotIndexedTest.testGroupingDVOnly(DocValuesNotIndexedTest.java:377)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 12405 lines...]
   [junit4] Suite: org.apache.solr.cloud.DocValuesNotIndexedTest
   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_B6E7B93B17A8657A-001/init-core-data-001
   [junit4]   2> 2139584 INFO  (SUITE-DocValuesNotIndexedTest-seed#[B6E7B93B17A8657A]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 2139585 INFO  (SUITE-DocValuesNotIndexedTest-seed#[B6E7B93B17A8657A]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth
   [junit4]   2> 2139624 INFO  (SUITE-DocValuesNotIndexedTest-seed#[B6E7B93B17A8657A]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 2139625 INFO  (SUITE-DocValuesNotIndexedTest-seed#[B6E7B93B17A8657A]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_B6E7B93B17A8657A-001/tempDir-001
   [junit4]   2> 2139625 INFO  (SUITE-DocValuesNotIndexedTest-seed#[B6E7B93B17A8657A]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 2139625 INFO  (Thread-3116) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 2139625 INFO  (Thread-3116) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 2139630 ERROR (Thread-3116) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 2139735 INFO  (SUITE-DocValuesNotIndexedTest-seed#[B6E7B93B17A8657A]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:52555
   [junit4]   2> 2139777 INFO  (jetty-launcher-5089-thread-2) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 2139777 INFO  (jetty-launcher-5089-thread-1) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 2139780 INFO  (jetty-launcher-5089-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@53923f71{/solr,null,AVAILABLE}
   [junit4]   2> 2139780 INFO  (jetty-launcher-5089-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7a90fd3f{/solr,null,AVAILABLE}
   [junit4]   2> 2139781 INFO  (jetty-launcher-5089-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@12f401c9{HTTP/1.1,[http/1.1]}{127.0.0.1:52558}
   [junit4]   2> 2139781 INFO  (jetty-launcher-5089-thread-1) [    ] o.e.j.s.Server Started @2146041ms
   [junit4]   2> 2139781 INFO  (jetty-launcher-5089-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=52558}
   [junit4]   2> 2139781 INFO  (jetty-launcher-5089-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@2701a0b7{HTTP/1.1,[http/1.1]}{127.0.0.1:52557}
   [junit4]   2> 2139781 ERROR (jetty-launcher-5089-thread-1) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2139781 INFO  (jetty-launcher-5089-thread-2) [    ] o.e.j.s.Server Started @2146041ms
   [junit4]   2> 2139782 INFO  (jetty-launcher-5089-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solrâ„¢ version 7.1.0
   [junit4]   2> 2139782 INFO  (jetty-launcher-5089-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=52557}
   [junit4]   2> 2139782 INFO  (jetty-launcher-5089-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2139782 INFO  (jetty-launcher-5089-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 2139782 INFO  (jetty-launcher-5089-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-08-18T20:25:24.549Z
   [junit4]   2> 2139783 ERROR (jetty-launcher-5089-thread-2) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2139783 INFO  (jetty-launcher-5089-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solrâ„¢ version 7.1.0
   [junit4]   2> 2139783 INFO  (jetty-launcher-5089-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2139783 INFO  (jetty-launcher-5089-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 2139783 INFO  (jetty-launcher-5089-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-08-18T20:25:24.550Z
   [junit4]   2> 2139785 INFO  (jetty-launcher-5089-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2139785 INFO  (jetty-launcher-5089-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2139793 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15df70594ac0001, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2139797 INFO  (jetty-launcher-5089-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52555/solr
   [junit4]   2> 2139797 INFO  (jetty-launcher-5089-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52555/solr
   [junit4]   2> 2139801 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15df70594ac0004, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2140038 INFO  (jetty-launcher-5089-thread-1) [n:127.0.0.1:52558_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2140038 INFO  (jetty-launcher-5089-thread-2) [n:127.0.0.1:52557_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2140040 INFO  (jetty-launcher-5089-thread-1) [n:127.0.0.1:52558_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:52558_solr
   [junit4]   2> 2140041 INFO  (jetty-launcher-5089-thread-2) [n:127.0.0.1:52557_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52557_solr
   [junit4]   2> 2140044 INFO  (jetty-launcher-5089-thread-1) [n:127.0.0.1:52558_solr    ] o.a.s.c.Overseer Overseer (id=98506370213216262-127.0.0.1:52558_solr-n_0000000000) starting
   [junit4]   2> 2140046 INFO  (zkCallback-5101-thread-1-processing-n:127.0.0.1:52558_solr) [n:127.0.0.1:52558_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2140046 INFO  (zkCallback-5100-thread-1-processing-n:127.0.0.1:52557_solr) [n:127.0.0.1:52557_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2140092 INFO  (jetty-launcher-5089-thread-1) [n:127.0.0.1:52558_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52558_solr
   [junit4]   2> 2140096 INFO  (zkCallback-5101-thread-1-processing-n:127.0.0.1:52558_solr) [n:127.0.0.1:52558_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2140097 INFO  (zkCallback-5100-thread-1-processing-n:127.0.0.1:52557_solr) [n:127.0.0.1:52557_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2140250 INFO  (jetty-launcher-5089-thread-1) [n:127.0.0.1:52558_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_52558.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1844b4e8
   [junit4]   2> 2140259 INFO  (jetty-launcher-5089-thread-1) [n:127.0.0.1:52558_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_52558.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1844b4e8
   [junit4]   2> 2140260 INFO  (jetty-launcher-5089-thread-1) [n:127.0.0.1:52558_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_52558.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1844b4e8
   [junit4]   2> 2140272 INFO  (jetty-launcher-5089-thread-1) [n:127.0.0.1:52558_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_B6E7B93B17A8657A-001/tempDir-001/node1/.
   [junit4]   2> 2140294 INFO  (jetty-launcher-5089-thread-2) [n:127.0.0.1:52557_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_52557.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1844b4e8
   [junit4]   2> 2140300 INFO  (jetty-launcher-5089-thread-2) [n:127.0.0.1:52557_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_52557.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1844b4e8
   [junit4]   2> 2140301 INFO  (jetty-launcher-5089-thread-2) [n:127.0.0.1:52557_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_52557.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1844b4e8
   [junit4]   2> 2140302 INFO  (jetty-launcher-5089-thread-2) [n:127.0.0.1:52557_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_B6E7B93B17A8657A-001/tempDir-001/node2/.
   [junit4]   2> 2140381 INFO  (SUITE-DocValuesNotIndexedTest-seed#[B6E7B93B17A8657A]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2140382 INFO  (SUITE-DocValuesNotIndexedTest-seed#[B6E7B93B17A8657A]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:52555/solr ready
   [junit4]   2> 2140394 INFO  (qtp1720180339-20681) [n:127.0.0.1:52558_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&maxShardsPerNode=2&name=dv_coll&nrtReplicas=1&action=CREATE&numShards=4&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2140397 INFO  (OverseerThreadFactory-7956-thread-1-processing-n:127.0.0.1:52558_solr) [n:127.0.0.1:52558_solr    ] o.a.s.c.CreateCollectionCmd Create collection dv_coll
   [junit4]   2> 2140516 INFO  (OverseerStateUpdate-98506370213216262-127.0.0.1:52558_solr-n_0000000000) [n:127.0.0.1:52558_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"dv_coll",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"dv_coll_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:52558/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 2140521 INFO  (OverseerStateUpdate-98506370213216262-127.0.0.1:52558_solr-n_0000000000) [n:127.0.0.1:52558_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"dv_coll",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"dv_coll_shard2_replica_n2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:52557/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 2140526 INFO  (OverseerStateUpdate-98506370213216262-127.0.0.1:52558_solr-n_0000000000) [n:127.0.0.1:52558_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"dv_coll",
   [junit4]   2>   "shard":"shard3",
   [junit4]   2>   "core":"dv_coll_shard3_replica_n4",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:52558/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 2140530 INFO  (OverseerStateUpdate-98506370213216262-127.0.0.1:52558_solr-n_0000000000) [n:127.0.0.1:52558_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"dv_coll",
   [junit4]   2>   "shard":"shard4",
   [junit4]   2>   "core":"dv_coll_shard4_replica_n6",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:52557/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 2140745 INFO  (qtp1720180339-20682) [n:127.0.0.1:52558_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf1&newCollection=true&name=dv_coll_shard1_replica_n1&action=CREATE&numShards=4&collection=dv_coll&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 2140746 INFO  (qtp1720180339-20683) [n:127.0.0.1:52558_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node7&collection.configName=conf1&newCollection=true&name=dv_coll_shard3_replica_n4&action=CREATE&numShards=4&collection=dv_coll&shard=shard3&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 2140747 INFO  (qtp1720180339-20682) [n:127.0.0.1:52558_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 2140747 INFO  (qtp1431678049-20685) [n:127.0.0.1:52557_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node8&collection.configName=conf1&newCollection=true&name=dv_coll_shard4_replica_n6&action=CREATE&numShards=4&collection=dv_coll&shard=shard4&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 2140748 INFO  (qtp1431678049-20687) [n:127.0.0.1:52557_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node5&collection.configName=conf1&newCollection=true&name=dv_coll_shard2_replica_n2&action=CREATE&numShards=4&collection=dv_coll&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 2140748 INFO  (qtp1431678049-20685) [n:127.0.0.1:52557_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 2140871 INFO  (zkCallback-5101-thread-1-processing-n:127.0.0.1:52558_solr) [n:127.0.0.1:52558_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2140871 INFO  (zkCallback-5100-thread-1-processing-n:127.0.0.1:52557_solr) [n:127.0.0.1:52557_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2140871 INFO  (zkCallback-5101-thread-2-processing-n:127.0.0.1:52558_solr) [n:127.0.0.1:52558_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2140872 INFO  (zkCallback-5100-thread-2-processing-n:127.0.0.1:52557_solr) [n:127.0.0.1:52557_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2141792 INFO  (qtp1431678049-20685) [n:127.0.0.1:52557_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 2141793 INFO  (qtp1720180339-20683) [n:127.0.0.1:52558_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 2141795 INFO  (qtp1431678049-20687) [n:127.0.0.1:52557_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 2141803 INFO  (qtp1720180339-20682) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 2141805 INFO  (qtp1431678049-20685) [n:127.0.0.1:52557_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.s.IndexSchema [dv_coll_shard4_replica_n6] Schema name=minimal
   [junit4]   2> 2141838 INFO  (qtp1431678049-20687) [n:127.0.0.1:52557_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.s.IndexSchema [dv_coll_shard2_replica_n2] Schema name=minimal
   [junit4]   2> 2141840 INFO  (qtp1720180339-20683) [n:127.0.0.1:52558_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.s.IndexSchema [dv_coll_shard3_replica_n4] Schema name=minimal
   [junit4]   2> 2141841 INFO  (qtp1431678049-20685) [n:127.0.0.1:52557_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2141841 INFO  (qtp1431678049-20685) [n:127.0.0.1:52557_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.c.CoreContainer Creating SolrCore 'dv_coll_shard4_replica_n6' using configuration from collection dv_coll, trusted=true
   [junit4]   2> 2141842 INFO  (qtp1431678049-20685) [n:127.0.0.1:52557_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_52557.solr.core.dv_coll.shard4.replica_n6' (registry 'solr.core.dv_coll.shard4.replica_n6') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1844b4e8
   [junit4]   2> 2141842 INFO  (qtp1431678049-20685) [n:127.0.0.1:52557_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 2141842 INFO  (qtp1431678049-20685) [n:127.0.0.1:52557_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.c.SolrCore [[dv_coll_shard4_replica_n6] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_B6E7B93B17A8657A-001/tempDir-001/node2/dv_coll_shard4_replica_n6], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_B6E7B93B17A8657A-001/tempDir-001/node2/./dv_coll_shard4_replica_n6/data/]
   [junit4]   2> 2141842 INFO  (qtp1720180339-20683) [n:127.0.0.1:52558_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2141842 INFO  (qtp1720180339-20683) [n:127.0.0.1:52558_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.c.CoreContainer Creating SolrCore 'dv_coll_shard3_replica_n4' using configuration from collection dv_coll, trusted=true
   [junit4]   2> 2141843 INFO  (qtp1720180339-20683) [n:127.0.0.1:52558_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_52558.solr.core.dv_coll.shard3.replica_n4' (registry 'solr.core.dv_coll.shard3.replica_n4') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1844b4e8
   [junit4]   2> 2141843 INFO  (qtp1720180339-20683) [n:127.0.0.1:52558_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 2141843 INFO  (qtp1720180339-20683) [n:127.0.0.1:52558_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.c.SolrCore [[dv_coll_shard3_replica_n4] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_B6E7B93B17A8657A-001/tempDir-001/node1/dv_coll_shard3_replica_n4], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_B6E7B93B17A8657A-001/tempDir-001/node1/./dv_coll_shard3_replica_n4/data/]
   [junit4]   2> 2141838 INFO  (qtp1720180339-20682) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.IndexSchema [dv_coll_shard1_replica_n1] Schema name=minimal
   [junit4]   2> 2141846 INFO  (qtp1431678049-20687) [n:127.0.0.1:52557_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2141846 INFO  (qtp1431678049-20687) [n:127.0.0.1:52557_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.c.CoreContainer Creating SolrCore 'dv_coll_shard2_replica_n2' using configuration from collection dv_coll, trusted=true
   [junit4]   2> 2141847 INFO  (qtp1720180339-20682) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2141847 INFO  (qtp1720180339-20682) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'dv_coll_shard1_replica_n1' using configuration from collection dv_coll, trusted=true
   [junit4]   2> 2141847 INFO  (qtp1431678049-20687) [n:127.0.0.1:52557_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_52557.solr.core.dv_coll.shard2.replica_n2' (registry 'solr.core.dv_coll.shard2.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1844b4e8
   [junit4]   2> 2141847 INFO  (qtp1431678049-20687) [n:127.0.0.1:52557_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 2141847 INFO  (qtp1431678049-20687) [n:127.0.0.1:52557_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.c.SolrCore [[dv_coll_shard2_replica_n2] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_B6E7B93B17A8657A-001/tempDir-001/node2/dv_coll_shard2_replica_n2], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_B6E7B93B17A8657A-001/tempDir-001/node2/./dv_coll_shard2_replica_n2/data/]
   [junit4]   2> 2141847 INFO  (qtp1720180339-20682) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_52558.solr.core.dv_coll.shard1.replica_n1' (registry 'solr.core.dv_coll.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1844b4e8
   [junit4]   2> 2141847 INFO  (qtp1720180339-20682) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 2141847 INFO  (qtp1720180339-20682) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.SolrCore [[dv_coll_shard1_replica_n1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_B6E7B93B17A8657A-001/tempDir-001/node1/dv_coll_shard1_replica_n1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_B6E7B93B17A8657A-001/tempDir-001/node1/./dv_coll_shard1_replica_n1/data/]
   [junit4]   2> 2141940 INFO  (qtp1720180339-20682) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2141940 INFO  (qtp1720180339-20682) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2141941 INFO  (qtp1720180339-20682) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2141941 INFO  (qtp1720180339-20682) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2141968 INFO  (qtp1720180339-20683) [n:127.0.0.1:52558_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2141968 INFO  (qtp1720180339-20683) [n:127.0.0.1:52558_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2141970 INFO  (qtp1431678049-20685) [n:127.0.0.1:52557_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2141970 INFO  (qtp1431678049-20685) [n:127.0.0.1:52557_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2141975 INFO  (qtp1720180339-20682) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@65cffefa[dv_coll_shard1_replica_n1] main]
   [junit4]   2> 2141975 INFO  (qtp1720180339-20683) [n:127.0.0.1:52558_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2141975 INFO  (qtp1720180339-20683) [n:127.0.0.1:52558_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2141976 INFO  (qtp1431678049-20685) [n:127.0.0.1:52557_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2141976 INFO  (qtp1431678049-20687) [n:127.0.0.1:52557_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2141976 INFO  (qtp1431678049-20685) [n:127.0.0.1:52557_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2141976 INFO  (qtp1431678049-20687) [n:127.0.0.1:52557_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2141977 INFO  (qtp1720180339-20682) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 2141977 INFO  (qtp1720180339-20682) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2141978 INFO  (qtp1720180339-20682) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/conf1/managed-schema
   [junit4]   2> 2141978 INFO  (qtp1431678049-20687) [n:127.0.0.1:52557_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2141978 INFO  (qtp1431678049-20687) [n:127.0.0.1:52557_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2141978 INFO  (qtp1720180339-20682) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
   [junit4]   2> 2141979 INFO  (qtp1431678049-20685) [n:127.0.0.1:52557_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.s.SolrIndexSearcher Opening [Searcher@70663ed9[dv_coll_shard4_replica_n6] main]
   [junit4]   2> 2141979 INFO  (qtp1720180339-20682) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 2141980 INFO  (qtp1431678049-20685) [n:127.0.0.1:52557_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 2141980 INFO  (searcherExecutor-7964-thread-1-processing-n:127.0.0.1:52558_solr x:dv_coll_shard1_replica_n1 s:shard1 c:dv_coll r:core_node3) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.SolrCore [dv_coll_shard1_replica_n1] Registered new searcher Searcher@65cffefa[dv_coll_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2141980 INFO  (qtp1720180339-20682) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1576101925876662272
   [junit4]   2> 2141980 INFO  (qtp1431678049-20685) [n:127.0.0.1:52557_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2141980 INFO  (qtp1431678049-20685) [n:127.0.0.1:52557_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/conf1/managed-schema
   [junit4]   2> 2141981 INFO  (qtp1431678049-20685) [n:127.0.0.1:52557_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
   [junit4]   2> 2141982 INFO  (qtp1431678049-20685) [n:127.0.0.1:52557_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 2141982 INFO  (qtp1431678049-20685) [n:127.0.0.1:52557_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1576101925878759424
   [junit4]   2> 2141982 INFO  (searcherExecutor-7961-thread-1-processing-n:127.0.0.1:52557_solr x:dv_coll_shard4_replica_n6 s:shard4 c:dv_coll r:core_node8) [n:127.0.0.1:52557_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.c.SolrCore [dv_coll_shard4_replica_n6] Registered new searcher Searcher@70663ed9[dv_coll_shard4_replica_n6] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2141983 INFO  (qtp1431678049-20687) [n:127.0.0.1:52557_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@53832de8[dv_coll_shard2_replica_n2] main]
   [junit4]   2> 2141983 INFO  (qtp1720180339-20683) [n:127.0.0.1:52558_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.s.SolrIndexSearcher Opening [Searcher@10b1b46d[dv_coll_shard3_replica_n4] main]
   [junit4]   2> 2141988 INFO  (qtp1431678049-20687) [n:127.0.0.1:52557_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 2141988 INFO  (qtp1431678049-20687) [n:127.0.0.1:52557_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2141989 INFO  (qtp1431678049-20687) [n:127.0.0.1:52557_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/conf1/managed-schema
   [junit4]   2> 2141990 INFO  (qtp1431678049-20687) [n:127.0.0.1:52557_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
   [junit4]   2> 2141991 INFO  (qtp1431678049-20685) [n:127.0.0.1:52557_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 2141991 INFO  (qtp1431678049-20685) [n:127.0.0.1:52557_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 2141991 INFO  (qtp1431678049-20685) [n:127.0.0.1:52557_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:52557/solr/dv_coll_shard4_replica_n6/
   [junit4]   2> 2141991 INFO  (qtp1431678049-20685) [n:127.0.0.1:52557_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 2141991 INFO  (qtp1431678049-20685) [n:127.0.0.1:52557_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.c.SyncStrategy http://127.0.0.1:52557/solr/dv_coll_shard4_replica_n6/ has no replicas
   [junit4]   2> 2141991 INFO  (qtp1431678049-20685) [n:127.0.0.1:52557_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 2141991 INFO  (qtp1720180339-20682) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 2141992 INFO  (qtp1720180339-20682) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 2141992 INFO  (qtp1720180339-20682) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:52558/solr/dv_coll_shard1_replica_n1/
   [junit4]   2> 2141992 INFO  (qtp1720180339-20682) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 2141992 INFO  (qtp1720180339-20682) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:52558/solr/dv_coll_shard1_replica_n1/ has no replicas
   [junit4]   2> 2141992 INFO  (qtp1720180339-20682) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 2141992 INFO  (qtp1431678049-20687) [n:127.0.0.1:52557_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 2141994 INFO  (searcherExecutor-7963-thread-1-processing-n:127.0.0.1:52557_solr x:dv_coll_shard2_replica_n2 s:shard2 c:dv_coll r:core_node5) [n:127.0.0.1:52557_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.c.SolrCore [dv_coll_shard2_replica_n2] Registered new searcher Searcher@53832de8[dv_coll_shard2_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2141994 INFO  (qtp1431678049-20687) [n:127.0.0.1:52557_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1576101925891342336
   [junit4]   2> 2141996 INFO  (qtp1431678049-20685) [n:127.0.0.1:52557_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:52557/solr/dv_coll_shard4_replica_n6/ shard4
   [junit4]   2> 2141997 INFO  (qtp1720180339-20683) [n:127.0.0.1:52558_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 2141999 INFO  (qtp1720180339-20683) [n:127.0.0.1:52558_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2141999 INFO  (qtp1720180339-20683) [n:127.0.0.1:52558_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/conf1/managed-schema
   [junit4]   2> 2142000 INFO  (qtp1720180339-20683) [n:127.0.0.1:52558_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
   [junit4]   2> 2142000 INFO  (qtp1720180339-20682) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:52558/solr/dv_coll_shard1_replica_n1/ shard1
   [junit4]   2> 2142003 INFO  (qtp1431678049-20687) [n:127.0.0.1:52557_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 2142003 INFO  (qtp1431678049-20687) [n:127.0.0.1:52557_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 2142003 INFO  (qtp1431678049-20687) [n:127.0.0.1:52557_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:52557/solr/dv_coll_shard2_replica_n2/
   [junit4]   2> 2142003 INFO  (qtp1431678049-20687) [n:127.0.0.1:52557_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 2142003 INFO  (qtp1431678049-20687) [n:127.0.0.1:52557_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.c.SyncStrategy http://127.0.0.1:52557/solr/dv_coll_shard2_replica_n2/ has no replicas
   [junit4]   2> 2142003 INFO  (qtp1431678049-20687) [n:127.0.0.1:52557_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 2142004 INFO  (qtp1720180339-20683) [n:127.0.0.1:52558_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 2142004 INFO  (searcherExecutor-7962-thread-1-processing-n:127.0.0.1:52558_solr x:dv_coll_shard3_replica_n4 s:shard3 c:dv_coll r:core_node7) [n:127.0.0.1:52558_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.c.SolrCore [dv_coll_shard3_replica_n4] Registered new searcher Searcher@10b1b46d[dv_coll_shard3_replica_n4] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2142005 INFO  (qtp1720180339-20683) [n:127.0.0.1:52558_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1576101925902876672
   [junit4]   2> 2142009 INFO  (qtp1431678049-20687) [n:127.0.0.1:52557_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:52557/solr/dv_coll_shard2_replica_n2/ shard2
   [junit4]   2> 2142015 INFO  (qtp1720180339-20683) [n:127.0.0.1:52558_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 2142015 INFO  (qtp1720180339-20683) [n:127.0.0.1:52558_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 2142015 INFO  (qtp1720180339-20683) [n:127.0.0.1:52558_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:52558/solr/dv_coll_shard3_replica_n4/
   [junit4]   2> 2142015 INFO  (qtp1720180339-20683) [n:127.0.0.1:52558_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 2142015 INFO  (qtp1720180339-20683) [n:127.0.0.1:52558_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.c.SyncStrategy http://127.0.0.1:52558/solr/dv_coll_shard3_replica_n4/ has no replicas
   [junit4]   2> 2142015 INFO  (qtp1720180339-20683) [n:127.0.0.1:52558_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 2142020 INFO  (qtp1720180339-20683) [n:127.0.0.1:52558_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:52558/solr/dv_coll_shard3_replica_n4/ shard3
   [junit4]   2> 2142124 INFO  (zkCallback-5101-thread-2-processing-n:127.0.0.1:52558_solr) [n:127.0.0.1:52558_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2142124 INFO  (zkCallback-5101-thread-1-processing-n:127.0.0.1:52558_solr) [n:127.0.0.1:52558_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2142124 INFO  (zkCallback-5100-thread-2-processing-n:127.0.0.1:52557_solr) [n:127.0.0.1:52557_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2142124 INFO  (zkCallback-5100-thread-1-processing-n:127.0.0.1:52557_solr) [n:127.0.0.1:52557_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2142132 INFO  (qtp1720180339-20683) [n:127.0.0.1:52558_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 2142134 INFO  (qtp1720180339-20683) [n:127.0.0.1:52558_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node7&collection.configName=conf1&newCollection=true&name=dv_coll_shard3_replica_n4&action=CREATE&numShards=4&collection=dv_coll&shard=shard3&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1388
   [junit4]   2> 2142178 INFO  (qtp1431678049-20685) [n:127.0.0.1:52557_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 2142178 INFO  (qtp1431678049-20687) [n:127.0.0.1:52557_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 2142178 INFO  (qtp1720180339-20682) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 2142180 INFO  (qtp1431678049-20685) [n:127.0.0.1:52557_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node8&collection.configName=conf1&newCollection=true&name=dv_coll_shard4_replica_n6&action=CREATE&numShards=4&collection=dv_coll&shard=shard4&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1432
   [junit4]   2> 2142181 INFO  (qtp1431678049-20687) [n:127.0.0.1:52557_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node5&collection.configName=conf1&newCollection=true&name=dv_coll_shard2_replica_n2&action=CREATE&numShards=4&collection=dv_coll&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1433
   [junit4]   2> 2142181 INFO  (qtp1720180339-20682) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf1&newCollection=true&name=dv_coll_shard1_replica_n1&action=CREATE&numShards=4&collection=dv_coll&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1435
   [junit4]   2> 2142185 INFO  (qtp1720180339-20681) [n:127.0.0.1:52558_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 2142292 INFO  (zkCallback-5100-thread-1-processing-n:127.0.0.1:52557_solr) [n:127.0.0.1:52557_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2142292 INFO  (zkCallback-5100-thread-2-processing-n:127.0.0.1:52557_solr) [n:127.0.0.1:52557_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2142292 INFO  (zkCallback-5101-thread-2-processing-n:127.0.0.1:52558_solr) [n:127.0.0.1:52558_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2142292 INFO  (zkCallback-5101-thread-1-processing-n:127.0.0.1:52558_solr) [n:127.0.0.1:52558_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2142408 INFO  (OverseerCollectionConfigSetProcessor-98506370213216262-127.0.0.1:52558_solr-n_0000000000) [n:127.0.0.1:52558_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 2143188 INFO  (qtp1720180339-20681) [n:127.0.0.1:52558_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&maxShardsPerNode=2&name=dv_coll&nrtReplicas=1&action=CREATE&numShards=4&wt=javabin&version=2} status=0 QTime=2793
   [junit4]   2> 2143200 INFO  (qtp1720180339-20686) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.SchemaManager managed schema loaded . version : 0 
   [junit4]   2> 2143205 INFO  (qtp1720180339-20686) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.IndexSchema [dv_coll_shard1_replica_n1] Schema name=minimal
   [junit4]   2> 2143209 INFO  (qtp1720180339-20686) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2143251 INFO  (qtp1720180339-20686) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.ZkController Persisted config data to node /configs/conf1/managed-schema 
   [junit4]   2> 2143251 INFO  (zkCallback-5101-thread-1-processing-n:127.0.0.1:52558_solr) [n:127.0.0.1:52558_solr    ] o.a.s.s.ZkIndexSchemaReader A schema change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/configs/conf1/managed-schema, has occurred - updating schema from ZooKeeper ...
   [junit4]   2> 2143252 INFO  (zkCallback-5101-thread-2-processing-n:127.0.0.1:52558_solr) [n:127.0.0.1:52558_solr    ] o.a.s.s.ZkIndexSchemaReader A schema change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/configs/conf1/managed-schema, has occurred - updating schema from ZooKeeper ...
   [junit4]   2> 2143253 INFO  (zkCallback-5100-thread-2-processing-n:127.0.0.1:52557_solr) [n:127.0.0.1:52557_solr    ] o.a.s.s.ZkIndexSchemaReader A schema change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/configs/conf1/managed-schema, has occurred - updating schema from ZooKeeper ...
   [junit4]   2> 2143253 INFO  (zkCallback-5100-thread-1-processing-n:127.0.0.1:52557_solr) [n:127.0.0.1:52557_solr    ] o.a.s.s.ZkIndexSchemaReader A schema change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/configs/conf1/managed-schema, has occurred - updating schema from ZooKeeper ...
   [junit4]   2> 2143255 INFO  (zkCallback-5101-thread-1-processing-n:127.0.0.1:52558_solr) [n:127.0.0.1:52558_solr    ] o.a.s.s.ZkIndexSchemaReader Retrieved schema version 1 from ZooKeeper
   [junit4]   2> 2143260 INFO  (zkCallback-5101-thread-1-processing-n:127.0.0.1:52558_solr) [n:127.0.0.1:52558_solr    ] o.a.s.s.IndexSchema [dv_coll_shard3_replica_n4] Schema name=minimal
   [junit4]   2> 2143263 INFO  (zkCallback-5100-thread-2-processing-n:127.0.0.1:52557_solr) [n:127.0.0.1:52557_solr    ] o.a.s.s.ZkIndexSchemaReader Retrieved schema version 1 from ZooKeeper
   [junit4]   2> 2143266 INFO  (zkCallback-5100-thread-2-processing-n:127.0.0.1:52557_solr) [n:127.0.0.1:52557_solr    ] o.a.s.s.IndexSchema [dv_coll_shard4_replica_n6] Schema name=minimal
   [junit4]   2> 2143276 INFO  (zkCallback-5100-thread-1-processing-n:127.0.0.1:52557_solr) [n:127.0.0.1:52557_solr    ] o.a.s.s.ZkIndexSchemaReader Retrieved schema version 1 from ZooKeeper
   [junit4]   2> 2143283 INFO  (zkCallback-5101-thread-1-processing-n:127.0.0.1:52558_solr) [n:127.0.0.1:52558_solr    ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2143283 INFO  (zkCallback-5101-thread-1-processing-n:127.0.0.1:52558_solr) [n:127.0.0.1:52558_solr    ] o.a.s.s.ZkIndexSchemaReader Finished refreshing schema in 28 ms
   [junit4]   2> 2143286 INFO  (zkCallback-5100-thread-2-processing-n:127.0.0.1:52557_solr) [n:127.0.0.1:52557_solr    ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2143286 INFO  (zkCallback-5100-thread-2-processing-n:127.0.0.1:52557_solr) [n:127.0.0.1:52557_solr    ] o.a.s.s.ZkIndexSchemaReader Finished refreshing schema in 22 ms
   [junit4]   2> 2143286 INFO  (zkCallback-5100-thread-1-processing-n:127.0.0.1:52557_solr) [n:127.0.0.1:52557_solr    ] o.a.s.s.IndexSchema [dv_coll_shard2_replica_n2] Schema name=minimal
   [junit4]   2> 2143288 INFO  (Thread-3135) [n:127.0.0.1:52558_solr    ] o.a.s.c.SolrCore config update listener called for core dv_coll_shard1_replica_n1
   [junit4]   2> 2143288 INFO  (Thread-3136) [n:127.0.0.1:52557_solr    ] o.a.s.c.SolrCore config update listener called for core dv_coll_shard4_replica_n6
   [junit4]   2> 2143291 INFO  (zkCallback-5100-thread-1-processing-n:127.0.0.1:52557_solr) [n:127.0.0.1:52557_solr    ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2143291 INFO  (zkCallback-5100-thread-1-processing-n:127.0.0.1:52557_solr) [n:127.0.0.1:52557_solr    ] o.a.s.s.ZkIndexSchemaReader Finished refreshing schema in 14 ms
   [junit4]   2> 2143293 INFO  (Thread-3135) [n:127.0.0.1:52558_solr    ] o.a.s.c.SolrCore core reload dv_coll_shard1_replica_n1
   [junit4]   2> 2143294 INFO  (Thread-3136) [n:127.0.0.1:52557_solr    ] o.a.s.c.SolrCore config update listener called for core dv_coll_shard2_replica_n2
   [junit4]   2> 2143303 INFO  (qtp1720180339-20686) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 2143375 INFO  (qtp1720180339-20686) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.IndexSchema [dv_coll_shard1_replica_n1] Schema name=minimal
   [junit4]   2> 2143381 INFO  (qtp1720180339-20686) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2143381 INFO  (qtp1720180339-20686) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.CoreContainer Reloading SolrCore 'dv_coll_shard1_replica_n1' using configuration from collection dv_coll
   [junit4]   2> 2143416 INFO  (qtp1720180339-20686) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_52558.solr.core.dv_coll.shard1.replica_n1' (registry 'solr.core.dv_coll.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1844b4e8
   [junit4]   2> 2143416 INFO  (qtp1720180339-20686) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.SolrCore [[dv_coll_shard1_replica_n1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_B6E7B93B17A8657A-001/tempDir-001/node1/dv_coll_shard1_replica_n1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_B6E7B93B17A8657A-001/tempDir-001/node1/./dv_coll_shard1_replica_n1/data/]
   [junit4]   2> 2143498 INFO  (qtp1720180339-20686) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2143499 INFO  (qtp1720180339-20686) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2143500 INFO  (qtp1720180339-20686) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6b578be9[dv_coll_shard1_replica_n1] main]
   [junit4]   2> 2143502 INFO  (qtp1720180339-20686) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 2143502 INFO  (qtp1720180339-20686) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2143502 INFO  (qtp1720180339-20686) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/conf1/managed-schema
   [junit4]   2> 2143503 INFO  (qtp1720180339-20686) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.ZkIndexSchemaReader Current schema version 1 is already the latest
   [junit4]   2> 2143503 INFO  (qtp1720180339-20686) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 2143504 INFO  (qtp1720180339-20686) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1576101927473643520
   [junit4]   2> 2143506 INFO  (searcherExecutor-7981-thread-1-processing-n:127.0.0.1:52558_solr x:dv_coll_shard1_replica_n1 s:shard1 c:dv_coll r:core_node3) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.SolrCore [dv_coll_shard1_replica_n1] Registered new searcher Searcher@6b578be9[dv_coll_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2143537 INFO  (qtp1720180339-20686) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
   [junit4]   2> 2143547 INFO  (qtp1720180339-20686) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@201c66b1[dv_coll_shard1_replica_n1] main]
   [junit4]   2> 2143548 INFO  (zkCallback-5101-thread-2-processing-n:127.0.0.1:52558_solr) [n:127.0.0.1:52558_solr    ] o.a.s.s.ZkIndexSchemaReader Retrieved schema version 1 from ZooKeeper
   [junit4]   2> 2143550 INFO  (searcherExecutor-7981-thread-1-processing-n:127.0.0.1:52558_solr x:dv_coll_shard1_replica_n1 s:shard1 c:dv_coll r:core_node3) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.SolrCore [dv_coll_shard1_replica_n1] Registered new searcher Searcher@201c66b1[dv_coll_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2143550 INFO  (qtp1720180339-20686) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.ManagedIndexSchema Waiting up to 599 secs for 3 replicas to apply schema update version 1 for collection dv_coll
   [junit4]   2> 2143553 INFO  (zkCallback-5101-thread-2-processing-n:127.0.0.1:52558_solr) [n:127.0.0.1:52558_solr    ] o.a.s.s.IndexSchema [dv_coll_shard1_replica_n1] Schema name=minimal
   [junit4]   2> 2143556 INFO  (Thread-3135) [n:127.0.0.1:52558_solr    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 2143557 INFO  (qtp1431678049-20689) [n:127.0.0.1:52557_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.c.S.Request [dv_coll_shard2_replica_n2]  webapp=/solr path=/schema/zkversion params={refreshIfBelowVersion=1&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 2143560 INFO  (zkCallback-5101-thread-2-processing-n:127.0.0.1:52558_solr) [n:127.0.0.1:52558_solr    ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2143560 INFO  (zkCallback-5101-thread-2-processing-n:127.0.0.1:52558_solr) [n:127.0.0.1:52558_solr    ] o.a.s.s.ZkIndexSchemaReader Finished refreshing schema in 11 ms
   [junit4]   2> 2143561 INFO  (qtp1720180339-20688) [n:127.0.0.1:52558_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.c.S.Request [dv_coll_shard3_replica_n4]  webapp=/solr path=/schema/zkversion params={refreshIfBelowVersion=1&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 2143564 INFO  (qtp1431678049-20694) [n:127.0.0.1:52557_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.c.S.Request [dv_coll_shard4_replica_n6]  webapp=/solr path=/schema/zkversion params={refreshIfBelowVersion=1&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 2143564 INFO  (Thread-3135) [n:127.0.0.1:52558_solr    ] o.a.s.s.IndexSchema [dv_coll_shard1_replica_n1] Schema name=minimal
   [junit4]   2> 2143565 INFO  (qtp1720180339-20686) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.ManagedIndexSchema Took 16.0ms for 3 replicas to apply schema update version 1 for collection dv_coll
   [junit4]   2> 2143565 INFO  (qtp1720180339-20686) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.S.Request [dv_coll_shard1_replica_n1]  webapp=/solr path=/schema params={wt=javabin&version=2} status=0 QTime=367
   [junit4]   2> 2143565 INFO  (qtp1720180339-20686) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.SolrCore [dv_coll_shard1_replica_n1]  CLOSING SolrCore org.apache.solr.core.SolrCore@44d35c7d
   [junit4]   2> 2143565 INFO  (qtp1720180339-20686) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.dv_coll.shard1.replica_n1, tag=1154702461
   [junit4]   2> 2143565 INFO  (qtp1720180339-20686) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@265b7d02: rootName = solr_52558, domain = solr.core.dv_coll.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.dv_coll.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@2956de4f
   [junit4]   2> 2143605 INFO  (Thread-3135) [n:127.0.0.1:52558_solr    ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2143605 INFO  (Thread-3135) [n:127.0.0.1:52558_solr    ] o.a.s.c.CoreContainer Reloading SolrCore 'dv_coll_shard1_replica_n1' using configuration from collection dv_coll
   [junit4]   2> 2143608 INFO  (qtp1720180339-20686) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.dv_coll.shard1.leader, tag=1154702461
   [junit4]   2> 2143666 INFO  (Thread-3135) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_52558.solr.core.dv_coll.shard1.replica_n1' (registry 'solr.core.dv_coll.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1844b4e8
   [junit4]   2> 2143666 INFO  (Thread-3135) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.SolrCore [[dv_coll_shard1_replica_n1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_B6E7B93B17A8657A-001/tempDir-001/node1/dv_coll_shard1_replica_n1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_B6E7B93B17A8657A-001/tempDir-001/node1/./dv_coll_shard1_replica_n1/data/]
   [junit4]   2> 2143691 INFO  (TEST-DocValuesNotIndexedTest.testGroupingDVOnly-seed#[B6E7B93B17A8657A]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testGroupingDVOnly
   [junit4]   2> 2143718 INFO  (qtp1431678049-20685) [n:127.0.0.1:52557_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard2_replica_n2]  webapp=/solr path=/update params={update.distrib=TOLEADER&distrib.from=http://127.0.0.1:52558/solr/dv_coll_shard3_replica_n4/&wt=javabin&version=2}{deleteByQuery=*:* (-1576101927686504448)} 0 11
   [junit4]   2> 2143747 INFO  (qtp1431678049-20692) [n:127.0.0.1:52557_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard4_replica_n6]  webapp=/solr path=/update params={update.distrib=TOLEADER&distrib.from=http://127.0.0.1:52558/solr/dv_coll_shard3_replica_n4/&wt=javabin&version=2}{deleteByQuery=*:* (-1576101927700135936)} 0 28
   [junit4]   2> 2143750 INFO  (Thread-3135) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2143750 INFO  (Thread-3135) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2143769 INFO  (Thread-3135) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@641d21dc[dv_coll_shard1_replica_n1] main]
   [junit4]   2> 2143770 INFO  (qtp1720180339-20682) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard1_replica_n1]  webapp=/solr path=/update params={update.distrib=TOLEADER&distrib.from=http://127.0.0.1:52558/solr/dv_coll_shard3_replica_n4/&wt=javabin&version=2}{deleteByQuery=*:* (-1576101927730544640)} 0 21
   [junit4]   2> 2143770 INFO  (qtp1720180339-20690) [n:127.0.0.1:52558_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard3_replica_n4]  webapp=/solr path=/update params={wt=javabin&version=2}{deleteByQuery=*:* (-1576101927672872960)} 0 78
   [junit4]   2> 2143773 INFO  (qtp1720180339-20681) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1576101927756759040,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 2143773 INFO  (qtp1720180339-20681) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 2143773 INFO  (qtp1431678049-20694) [n:127.0.0.1:52557_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1576101927756759040,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 2143773 INFO  (qtp1431678049-20694) [n:127.0.0.1:52557_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 2143773 INFO  (qtp1720180339-20688) [n:127.0.0.1:52558_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1576101927756759040,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 2143773 INFO  (qtp1720180339-20688) [n:127.0.0.1:52558_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 2143773 INFO  (Thread-3135) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 2143774 INFO  (qtp1431678049-20678) [n:127.0.0.1:52557_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1576101927757807616,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 2143774 INFO  (qtp1431678049-20678) [n:127.0.0.1:52557_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 2143774 INFO  (Thread-3135) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2143774 INFO  (Thread-3135) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/conf1/managed-schema
   [junit4]   2> 2143782 INFO  (Thread-3135) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.ZkIndexSchemaReader Current schema version 1 is already the latest
   [junit4]   2> 2143782 INFO  (Thread-3135) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 2143783 INFO  (qtp1720180339-20681) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 2143783 INFO  (qtp1720180339-20681) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard1_replica_n1]  webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:52557/solr/dv_coll_shard4_replica_n6/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 10
   [junit4]   2> 2143784 INFO  (qtp1720180339-20688) [n:127.0.0.1:52558_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.s.SolrIndexSearcher Opening [Searcher@19e3fd5a[dv_coll_shard3_replica_n4] main]
   [junit4]   2> 2143784 INFO  (searcherExecutor-7986-thread-1-processing-n:127.0.0.1:52558_solr x:dv_coll_shard1_replica_n1 s:shard1 c:dv_coll r:core_node3) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.SolrCore [dv_coll_shard1_replica_n1] Registered new searcher Searcher@641d21dc[dv_coll_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2143784 INFO  (qtp1431678049-20694) [n:127.0.0.1:52557_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@62856834[dv_coll_shard2_replica_n2] main]
   [junit4]   2> 2143784 INFO  (qtp1720180339-20688) [n:127.0.0.1:52558_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 2143784 INFO  (qtp1431678049-20694) [n:127.0.0.1:52557_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 2143785 INFO  (qtp1431678049-20678) [n:127.0.0.1:52557_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.s.SolrIndexSearcher Opening [Searcher@32b817ff[dv_coll_shard4_replica_n6] main]
   [junit4]   2> 2143785 INFO  (searcherExecutor-7962-thread-1-processing-n:127.0.0.1:52558_solr x:dv_coll_shard3_replica_n4 s:shard3 c:dv_coll r:core_node7) [n:127.0.0.1:52558_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.c.SolrCore [dv_coll_shard3_replica_n4] Registered new searcher Searcher@19e3fd5a[dv_coll_shard3_replica_n4] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2143785 INFO  (qtp1431678049-20678) [n:127.0.0.1:52557_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 2143785 INFO  (qtp1720180339-20688) [n:127.0.0.1:52558_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard3_replica_n4]  webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:52557/solr/dv_coll_shard4_replica_n6/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 12
   [junit4]   2> 2143785 INFO  (searcherExecutor-7963-thread-1-processing-n:127.0.0.1:52557_solr x:dv_coll_shard2_replica_n2 s:shard2 c:dv_coll r:core_node5) [n:127.0.0.1:52557_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.c.SolrCore [dv_coll_shard2_replica_n2] Registered new searcher Searcher@62856834[dv_coll_shard2_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2143785 INFO  (qtp1431678049-20694) [n:127.0.0.1:52557_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard2_replica_n2]  webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:52557/solr/dv_coll_shard4_replica_n6/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 12
   [junit4]   2> 2143786 INFO  (searcherExecutor-7961-thread-1-processing-n:127.0.0.1:52557_solr x:dv_coll_shard4_replica_n6 s:shard4 c:dv_coll r:core_node8) [n:127.0.0.1:52557_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.c.SolrCore [dv_coll_shard4_replica_n6] Registered new searcher Searcher@32b817ff[dv_coll_shard4_replica_n6] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2143786 INFO  (qtp1431678049-20678) [n:127.0.0.1:52557_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard4_replica_n6]  webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:52557/solr/dv_coll_shard4_replica_n6/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 12
   [junit4]   2> 2143786 INFO  (qtp1431678049-20689) [n:127.0.0.1:52557_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard4_replica_n6]  webapp=/solr path=/update params={_stateVer_=dv_coll:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 15
   [junit4]   2> 2143791 INFO  (qtp1431678049-20691) [n:127.0.0.1:52557_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard4_replica_n6]  webapp=/solr path=/update params={_stateVer_=dv_coll:4&wt=javabin&version=2}{add=[10000 (1576101927773536256), 5 (1576101927774584832), 17 (1576101927775633408), 19 (1576101927775633409), 21 (1576101927775633410), 23 (1576101927775633411), 25 (1576101927775633412), 29 (1576101927775633413), 10030 (1576101927775633414), 33 (1576101927775633415), ... (13 adds)]} 0 2
   [junit4]   2> 2143791 INFO  (qtp1431678049-20687) [n:127.0.0.1:52557_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard2_replica_n2]  webapp=/solr path=/update params={_stateVer_=dv_coll:4&wt=javabin&version=2}{add=[0 (1576101927773536256), 4 (1576101927774584832), 10005 (1576101927775633408), 12 (1576101927775633409), 15 (1576101927775633410), 16 (1576101927775633411), 20 (1576101927775633412), 22 (1576101927775633413), 34 (1576101927775633414), 35 (1576101927775633415), ... (16 adds)]} 0 2
   [junit4]   2> 2143793 INFO  (qtp1720180339-20683) [n:127.0.0.1:52558_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard3_replica_n4]  webapp=/solr path=/update params={_stateVer_=dv_coll:4&wt=javabin&version=2}{add=[2 (1576101927776681984), 3 (1576101927777730560), 6 (1576101927777730561), 7 (1576101927777730562), 9 (1576101927777730563), 10010 (1576101927777730564), 18 (1576101927777730565), 10025 (1576101927777730566), 26 (1576101927777730567), 30 (1576101927777730568), ... (14 adds)]} 0 1
   [junit4]   2> 2143839 INFO  (Thread-3135) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
   [junit4]   2> 2143860 INFO  (qtp1720180339-20679) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard1_replica_n1]  webapp=/solr path=/update params={_stateVer_=dv_coll:4&wt=javabin&version=2}{add=[1 (1576101927773536256), 8 (1576101927831207936), 10 (1576101927846936576), 11 (1576101927846936577), 13 (1576101927847985152), 14 (1576101927847985153), 10015 (1576101927847985154), 10020 (1576101927847985155), 24 (1576101927847985156), 27 (1576101927847985157), ... (16 adds)]} 0 71
   [junit4]   2> 2143861 INFO  (qtp1431678049-20685) [n:127.0.0.1:52557_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1576101927849033728,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 2143861 INFO  (qtp1431678049-20685) [n:127.0.0.1:52557_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@68159bdd commitCommandVersion:1576101927849033728
   [junit4]   2> 2143861 INFO  (qtp1720180339-20690) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1576101927849033728,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 2143862 INFO  (qtp1720180339-20690) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@55c77ce3 commitCommandVersion:1576101927849033728
   [junit4]   2> 2143862 INFO  (qtp1720180339-20681) [n:127.0.0.1:52558_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1576101927850082304,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 2143862 INFO  (qtp1720180339-20681) [n:127.0.0.1:52558_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@70ef8ef commitCommandVersion:1576101927850082304
   [junit4]   2> 2143863 INFO  (qtp1431678049-20694) [n:127.0.0.1:52557_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1576101927851130880,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 2143863 INFO  (qtp1431678049-20694) [n:127.0.0.1:52557_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@53ade8d4 commitCommandVersion:1576101927851130880
   [junit4]   2> 2143928 INFO  (Thread-3135) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@2393a864[dv_coll_shard1_replica_n1] main]
   [junit4]   2> 2143928 INFO  (Thread-3135) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.SolrCore config update listener called for core dv_coll_shard3_replica_n4
   [junit4]   2> 2143930 INFO  (searcherExecutor-7986-thread-1-processing-n:127.0.0.1:52558_solr x:dv_coll_shard1_replica_n1 s:shard1 c:dv_coll r:core_node3) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.SolrCore [dv_coll_shard1_replica_n1] Registered new searcher Searcher@2393a864[dv_coll_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.1.0):C1)))}
   [junit4]   2> 2143991 INFO  (qtp1431678049-20694) [n:127.0.0.1:52557_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.s.SolrIndexSearcher Opening [Searcher@372a3c06[dv_coll_shard4_replica_n6] main]
   [junit4]   2> 2143991 INFO  (qtp1431678049-20694) [n:127.0.0.1:52557_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 2143992 INFO  (qtp1720180339-20681) [n:127.0.0.1:52558_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.s.SolrIndexSearcher Opening [Searcher@5da82f5b[dv_coll_shard3_replica_n4] main]
   [junit4]   2> 2143992 INFO  (qtp1720180339-20681) [n:127.0.0.1:52558_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 2143992 INFO  (qtp1431678049-20685) [n:127.0.0.1:52557_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@4afce838[dv_coll_shard2_replica_n2] main]
   [junit4]   2> 2143992 INFO  (searcherExecutor-7961-thread-1-processing-n:127.0.0.1:52557_solr x:dv_coll_shard4_replica_n6 s:shard4 c:dv_coll r:core_node8) [n:127.0.0.1:52557_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.c.SolrCore [dv_coll_shard4_replica_n6] Registered new searcher Searcher@372a3c06[dv_coll_shard4_replica_n6] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.1.0):C13)))}
   [junit4]   2> 2143993 INFO  (qtp1431678049-20694) [n:127.0.0.1:52557_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard4_replica_n6]  webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:52558/solr/dv_coll_shard3_replica_n4/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 130
   [junit4]   2> 2143993 INFO  (searcherExecutor-7962-thread-1-processing-n:127.0.0.1:52558_solr x:dv_coll_shard3_replica_n4 s:shard3 c:dv_coll r:core_node7) [n:127.0.0.1:52558_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.c.SolrCore [dv_coll_shard3_replica_n4] Registered new searcher Searcher@5da82f5b[dv_coll_shard3_replica_n4] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.1.0):C14)))}
   [junit4]   2> 2143993 INFO  (qtp1720180339-20681) [n:127.0.0.1:52558_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard3_replica_n4]  webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:52558/solr/dv_coll_shard3_replica_n4/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 131
   [junit4]   2> 2143993 INFO  (qtp1431678049-20685) [n:127.0.0.1:52557_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 2143993 INFO  (searcherExecutor-7963-thread-1-processing-n:127.0.0.1:52557_solr x:dv_coll_shard2_replica_n2 s:shard2 c:dv_coll r:core_node5) [n:127.0.0.1:52557_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.c.SolrCore [dv_coll_shard2_replica_n2] Registered new searcher Searcher@4afce838[dv_coll_shard2_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.1.0):C16)))}
   [junit4]   2> 2143994 INFO  (qtp1431678049-20685) [n:127.0.0.1:52557_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard2_replica_n2]  webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:52558/solr/dv_coll_shard3_replica_n4/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 132
   [junit4]   2> 2144016 INFO  (qtp1720180339-20690) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@62bd2c0c[dv_coll_shard1_replica_n1] main]
   [junit4]   2> 2144017 INFO  (qtp1720180339-20690) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 2144019 INFO  (searcherExecutor-7981-thread-1-processing-n:127.0.0.1:52558_solr x:dv_coll_shard1_replica_n1 s:shard1 c:dv_coll r:core_node3) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.SolrCore [dv_coll_shard1_replica_n1] Registered new searcher Searcher@62bd2c0c[dv_coll_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.1.0):C1) Uninverting(_1(7.1.0):C15)))}
   [junit4]   2> 2144019 INFO  (qtp1720180339-20690) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard1_replica_n1]  webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:52558/solr/dv_coll_shard3_replica_n4/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 158
   [junit4]   2> 2144019 INFO  (qtp1720180339-20690) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.SolrCore [dv_coll_shard1_replica_n1]  CLOSING SolrCore org.apache.solr.core.SolrCore@6c9f187a
   [junit4]   2> 2144019 INFO  (qtp1720180339-20690) [n:127.0.0.1:52558_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for reg

[...truncated too long message...]

1c55929: rootName = solr_52557, domain = solr.core.dv_coll.shard2.replica_n2, service url = null, agent id = null] for registry solr.core.dv_coll.shard2.replica_n2 / com.codahale.metrics.MetricRegistry@54b81ee1
   [junit4]   2> 2144547 INFO  (coreCloseExecutor-7990-thread-2) [n:127.0.0.1:52558_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.dv_coll.shard3.leader, tag=746170240
   [junit4]   2> 2144552 INFO  (coreCloseExecutor-7991-thread-2) [n:127.0.0.1:52557_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.dv_coll.shard2.leader, tag=1221053441
   [junit4]   2> 2144580 INFO  (jetty-closer-5090-thread-2) [    ] o.a.s.c.Overseer Overseer (id=98506370213216262-127.0.0.1:52558_solr-n_0000000000) closing
   [junit4]   2> 2144580 INFO  (OverseerStateUpdate-98506370213216262-127.0.0.1:52558_solr-n_0000000000) [n:127.0.0.1:52558_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:52558_solr
   [junit4]   2> 2144582 WARN  (zkCallback-5101-thread-3-processing-n:127.0.0.1:52558_solr) [n:127.0.0.1:52558_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 2144582 INFO  (jetty-closer-5090-thread-2) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@53923f71{/solr,null,UNAVAILABLE}
   [junit4]   2> 2144582 INFO  (zkCallback-5100-thread-3-processing-n:127.0.0.1:52557_solr) [n:127.0.0.1:52557_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
   [junit4]   2> 2144582 INFO  (zkCallback-5100-thread-1-processing-n:127.0.0.1:52557_solr) [n:127.0.0.1:52557_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:52557_solr
   [junit4]   2> 2146089 WARN  (zkCallback-5100-thread-1-processing-n:127.0.0.1:52557_solr) [n:127.0.0.1:52557_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 2146089 INFO  (jetty-closer-5090-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@7a90fd3f{/solr,null,UNAVAILABLE}
   [junit4]   2> 2146091 ERROR (SUITE-DocValuesNotIndexedTest-seed#[B6E7B93B17A8657A]-worker) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 2146091 INFO  (SUITE-DocValuesNotIndexedTest-seed#[B6E7B93B17A8657A]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:52555 52555
   [junit4]   2> 2146111 INFO  (Thread-3116) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:52555 52555
   [junit4]   2> 2146377 WARN  (Thread-3116) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	6	/solr/configs/conf1/managed-schema
   [junit4]   2> 	4	/solr/configs/conf1
   [junit4]   2> 	3	/solr/aliases.json
   [junit4]   2> 	3	/solr/clusterprops.json
   [junit4]   2> 	2	/solr/security.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	12	/solr/collections/dv_coll/state.json
   [junit4]   2> 	3	/solr/clusterstate.json
   [junit4]   2> 	2	/solr/configs/conf1/managed-schema
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	3	/solr/live_nodes
   [junit4]   2> 	3	/solr/collections
   [junit4]   2> 
   [junit4]   2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_B6E7B93B17A8657A-001
   [junit4]   2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarity(queryNorm=true): {}, locale=ga-IE, timezone=America/Knox_IN
   [junit4]   2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=3,threads=1,free=190768392,total=518979584
   [junit4]   2> NOTE: All tests run in this JVM: [TestStressLiveNodes, LeaderElectionContextKeyTest, TestInitQParser, FieldMutatingUpdateProcessorTest, TestStressCloudBlindAtomicUpdates, TestSchemaResource, TestEmbeddedSolrServerConstructors, HdfsNNFailoverTest, PingRequestHandlerTest, SolrIndexMetricsTest, TestSolrIndexConfig, TestReload, TestLegacyNumericUtils, SimpleMLTQParserTest, TestUseDocValuesAsStored, TestLRUCache, DistributedMLTComponentTest, LoggingHandlerTest, TestConfigSetsAPI, LeaderFailureAfterFreshStartTest, TestRealTimeGet, TestDocumentBuilder, TestRangeQuery, TestDFISimilarityFactory, TestCloudPseudoReturnFields, PreAnalyzedFieldManagedSchemaCloudTest, TestConfig, TestCoreDiscovery, CachingDirectoryFactoryTest, WrapperMergePolicyFactoryTest, SolrIndexConfigTest, TestDocSet, BigEndianAscendingWordDeserializerTest, SolrCLIZkUtilsTest, TestSchemaVersionResource, ShardRoutingCustomTest, AutoCommitTest, CacheHeaderTest, BasicFunctionalityTest, CdcrReplicationHandlerTest, TestComplexPhraseQParserPlugin, SolrMetricsIntegrationTest, BadCopyFieldTest, HighlighterMaxOffsetTest, PropertiesRequestHandlerTest, SynonymTokenizerTest, CloneFieldUpdateProcessorFactoryTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, BitVectorTest, HdfsDirectoryTest, MissingSegmentRecoveryTest, ShufflingReplicaListTransformerTest, TestFastLRUCache, MergeStrategyTest, AtomicUpdatesTest, TestCopyFieldCollectionResource, ShowFileRequestHandlerTest, TestJsonFacetRefinement, DistanceUnitsTest, TestSuggestSpellingConverter, TestPostingsSolrHighlighter, IndexSchemaRuntimeFieldTest, TestPerFieldSimilarity, TestCollationFieldDocValues, TestPerFieldSimilarityWithDefaultOverride, ClassificationUpdateProcessorIntegrationTest, TestIndexSearcher, TestJettySolrRunner, SortSpecParsingTest, BasicZkTest, TestSizeLimitedDistributedMap, ConvertedLegacyTest, TestSolrConfigHandler, LukeRequestHandlerTest, PluginInfoTest, PreAnalyzedFieldTest, ActionThrottleTest, TestClusterStateMutator, BasicDistributedZk2Test, TestXmlQParser, TestSegmentSorting, PreAnalyzedUpdateProcessorTest, TestReplicaProperties, TestCharFilters, ChangedSchemaMergeTest, TestSolrJ, TestReloadDeadlock, TestAnalyzedSuggestions, RestartWhileUpdatingTest, SolrGraphiteReporterTest, TestInfoStreamLogging, EchoParamsTest, BasicAuthStandaloneTest, AnalysisErrorHandlingTest, TestDefaultStatsCache, TestFieldCacheWithThreads, ZkSolrClientTest, TestGraphMLResponseWriter, TlogReplayBufferedWhileIndexingTest, TestJoin, SolrCmdDistributorTest, TestManagedSynonymFilterFactory, ExitableDirectoryReaderTest, UtilsToolTest, TestPrepRecovery, DistributedFacetPivotSmallTest, TestCollectionAPI, ReplaceNodeTest, DateRangeFieldTest, InfixSuggestersTest, TestSQLHandler, CollectionsAPISolrJTest, UpdateParamsTest, TestCloudJSONFacetJoinDomain, JsonLoaderTest, TestDynamicFieldResource, TestCollationField, DistributedTermsComponentTest, TestSolrFieldCacheBean, HdfsTlogReplayBufferedWhileIndexingTest, TestUninvertingReader, DebugComponentTest, ConfigSetsAPITest, CreateCollectionCleanupTest, TestSolrCloudWithHadoopAuthPlugin, TestNRTOpen, TestStressUserVersions, SpellCheckCollatorTest, LeaderInitiatedRecoveryOnShardRestartTest, TestMergePolicyConfig, LeaderElectionIntegrationTest, TestSolrDeletionPolicy1, TestConfigsApi, CollectionsAPIDistributedZkTest, TestRawTransformer, TestTolerantUpdateProcessorCloud, TestConfigSetsAPIZkFailure, CdcrVersionReplicationTest, TestHttpShardHandlerFactory, TestCloudSearcherWarming, SharedFSAutoReplicaFailoverUtilsTest, TestRandomRequestDistribution, TestSolr4Spatial2, VMParamsZkACLAndCredentialsProvidersTest, TestDelegationWithHadoopAuth, DistributedIntervalFacetingTest, BlockJoinFacetSimpleTest, TestSolrCloudWithKerberosAlt, HLLUtilTest, TestExportWriter, HdfsSyncSliceTest, TestManagedSchema, AsyncCallRequestStatusResponseTest, QueryEqualityTest, TestShardHandlerFactory, CdcrBootstrapTest, TestFieldTypeResource, TestAnalyzeInfixSuggestions, StatelessScriptUpdateProcessorFactoryTest, SegmentsInfoRequestHandlerTest, TestCrossCoreJoin, CleanupOldIndexTest, CursorMarkTest, DistribCursorPagingTest, SearchHandlerTest, TestSearchPerf, TestSolrCloudSnapshots, TestTrie, BlockDirectoryTest, PolyFieldTest, UpdateLogTest, TestBinaryResponseWriter, ZkStateWriterTest, ExternalFileFieldSortTest, HttpSolrCallGetCoreTest, TemplateUpdateProcessorTest, HdfsChaosMonkeySafeLeaderTest, TolerantUpdateProcessorTest, TestRestManager, TestCollapseQParserPlugin, ChaosMonkeySafeLeaderTest, UnloadDistributedZkTest, OpenCloseCoreStressTest, OverseerTest, FullSolrCloudDistribCmdsTest, TestRandomFaceting, TestZkChroot, TestRandomDVFaceting, ZkCLITest, TestDistributedSearch, TestDistributedGrouping, TestRecovery, TestStressReorder, TestStressVersions, HardAutoCommitTest, TestSolr4Spatial, StatsComponentTest, BadIndexSchemaTest, TestSort, TestFiltering, TestBadConfig, HighlighterTest, DistributedQueryElevationComponentTest, AnalysisAfterCoreReloadTest, SuggesterFSTTest, TestFoldingMultitermQuery, SuggesterTest, TestCSVLoader, SolrCoreCheckLockOnStartupTest, TestUpdate, DirectUpdateHandlerOptimizeTest, TestRemoteStreaming, SolrInfoBeanTest, IndexBasedSpellCheckerTest, TestWriterPerf, FileBasedSpellCheckerTest, DocumentBuilderTest, PathHierarchyTokenizerFactoryTest, MoreLikeThisHandlerTest, TestSolrQueryParser, RegexBoostProcessorTest, IndexSchemaTest, UpdateRequestProcessorFactoryTest, MBeansHandlerTest, TestPartialUpdateDeduplication, TestComponentsName, SOLR749Test, AlternateDirectoryTest, ResponseLogComponentTest, TestStressRecovery, ChaosMonkeySafeLeaderWithPullReplicasTest, CloudExitableDirectoryReaderTest, CollectionTooManyReplicasTest, DeleteReplicaTest, DistributedQueueTest, DocValuesNotIndexedTest]
   [junit4] Completed [509/729 (1!)] on J1 in 6.81s, 4 tests, 1 failure <<< FAILURES!

[...truncated 44087 lines...]