You are viewing a plain text version of this content. The canonical link for it is here.
Posted to builds@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2019/12/02 16:51:24 UTC

[JENKINS] Lucene-Solr-NightlyTests-8.x - Build # 286 - Unstable

Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-8.x/286/

1 tests failed.
FAILED:  org.apache.solr.cloud.RollingRestartTest.test

Error Message:
Timeout occurred while waiting response from server at: http://127.0.0.1:39471/btbk

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occurred while waiting response from server at: http://127.0.0.1:39471/btbk
	at __randomizedtesting.SeedInfo.seed([6537596EC6089843:ED6366B468F4F5BB]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:676)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:265)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:368)
	at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:296)
	at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.sendRequest(BaseCloudSolrClient.java:1127)
	at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.requestWithRetryOnStaleState(BaseCloudSolrClient.java:896)
	at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.request(BaseCloudSolrClient.java:828)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:228)
	at org.apache.solr.cloud.RollingRestartTest.restartWithRolesTest(RollingRestartTest.java:74)
	at org.apache.solr.cloud.RollingRestartTest.test(RollingRestartTest.java:53)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1082)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1054)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:171)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:564)
	... 52 more




Build Log:
[...truncated 15204 lines...]
   [junit4] Suite: org.apache.solr.cloud.RollingRestartTest
   [junit4]   2> 2482849 INFO  (SUITE-RollingRestartTest-seed#[6537596EC6089843]-worker) [     ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 2482849 INFO  (SUITE-RollingRestartTest-seed#[6537596EC6089843]-worker) [     ] o.a.s.SolrTestCaseJ4 Created dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_6537596EC6089843-001/data-dir-168-001
   [junit4]   2> 2482849 WARN  (SUITE-RollingRestartTest-seed#[6537596EC6089843]-worker) [     ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=100 numCloses=100
   [junit4]   2> 2482850 INFO  (SUITE-RollingRestartTest-seed#[6537596EC6089843]-worker) [     ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 2482851 INFO  (SUITE-RollingRestartTest-seed#[6537596EC6089843]-worker) [     ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 2482851 INFO  (SUITE-RollingRestartTest-seed#[6537596EC6089843]-worker) [     ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /btbk/
   [junit4]   2> 2482854 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 2482855 INFO  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 2482855 INFO  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 2482955 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.ZkTestServer start zk server on port:34263
   [junit4]   2> 2482955 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.ZkTestServer waitForServerUp: 127.0.0.1:34263
   [junit4]   2> 2482955 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:34263
   [junit4]   2> 2482955 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 34263
   [junit4]   2> 2482956 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2482959 INFO  (zkConnectionManagerCallback-12049-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2482959 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2482961 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2482962 INFO  (zkConnectionManagerCallback-12051-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2482962 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2482963 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 2482964 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 2482965 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 2482966 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 2482967 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 2482968 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 2482968 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 2482969 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 2482970 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 2482970 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 2482971 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 2482972 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
   [junit4]   2> 2483088 WARN  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 2483088 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 2483088 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 2483088 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.e.j.s.Server jetty-9.4.19.v20190610; built: 2019-06-10T16:30:51.723Z; git: afcf563148970e98786327af5e07c261fda175d3; jvm 1.8.0_191-b12
   [junit4]   2> 2483089 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 2483089 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 2483089 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 2483090 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@72e17981{/btbk,null,AVAILABLE}
   [junit4]   2> 2483090 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.e.j.s.AbstractConnector Started ServerConnector@24fb553c{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:38955}
   [junit4]   2> 2483090 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.e.j.s.Server Started @2483119ms
   [junit4]   2> 2483090 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_6537596EC6089843-001/tempDir-001/control/data, replicaType=NRT, hostContext=/btbk, hostPort=38955, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_6537596EC6089843-001/control-001/cores}
   [junit4]   2> 2483090 ERROR (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2483090 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 2483091 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.4.0
   [junit4]   2> 2483091 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2483091 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2483091 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-12-02T15:29:16.355Z
   [junit4]   2> 2483092 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2483092 INFO  (zkConnectionManagerCallback-12053-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2483092 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2483193 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 2483193 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_6537596EC6089843-001/control-001/solr.xml
   [junit4]   2> 2483196 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 2483196 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 2483197 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@2b3715f0, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 2483241 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
   [junit4]   2> 2483241 WARN  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@2b15da98[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2483241 WARN  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@2b15da98[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2483244 WARN  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@6b28638a[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2483244 WARN  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@6b28638a[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2483245 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34263/solr
   [junit4]   2> 2483246 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2483246 INFO  (zkConnectionManagerCallback-12060-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2483246 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2483348 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [n:127.0.0.1:38955_btbk     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2483349 INFO  (zkConnectionManagerCallback-12062-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2483349 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [n:127.0.0.1:38955_btbk     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2483404 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [n:127.0.0.1:38955_btbk     ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:38955_btbk
   [junit4]   2> 2483404 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [n:127.0.0.1:38955_btbk     ] o.a.s.c.Overseer Overseer (id=72825816818712580-127.0.0.1:38955_btbk-n_0000000000) starting
   [junit4]   2> 2483408 INFO  (OverseerStateUpdate-72825816818712580-127.0.0.1:38955_btbk-n_0000000000) [n:127.0.0.1:38955_btbk     ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:38955_btbk
   [junit4]   2> 2483409 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [n:127.0.0.1:38955_btbk     ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38955_btbk
   [junit4]   2> 2483409 INFO  (OverseerStateUpdate-72825816818712580-127.0.0.1:38955_btbk-n_0000000000) [n:127.0.0.1:38955_btbk     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2483427 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [n:127.0.0.1:38955_btbk     ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 2483439 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [n:127.0.0.1:38955_btbk     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2b3715f0
   [junit4]   2> 2483445 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [n:127.0.0.1:38955_btbk     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2b3715f0
   [junit4]   2> 2483445 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [n:127.0.0.1:38955_btbk     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2b3715f0
   [junit4]   2> 2483446 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [n:127.0.0.1:38955_btbk     ] o.a.s.p.PackageLoader /packages.json updated to version -1
   [junit4]   2> 2483447 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [n:127.0.0.1:38955_btbk     ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_6537596EC6089843-001/control-001/cores
   [junit4]   2> 2483460 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2483461 INFO  (zkConnectionManagerCallback-12071-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2483461 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2483462 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2483462 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:34263/solr ready
   [junit4]   2> 2483463 INFO  (qtp774187871-55536) [n:127.0.0.1:38955_btbk     ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:38955_btbk&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2483466 INFO  (OverseerThreadFactory-11430-thread-1-processing-n:127.0.0.1:38955_btbk) [n:127.0.0.1:38955_btbk     ] o.a.s.c.a.c.CreateCollectionCmd Create collection control_collection
   [junit4]   2> 2483570 INFO  (qtp774187871-55537) [n:127.0.0.1:38955_btbk    x:control_collection_shard1_replica_n1 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 2483570 INFO  (qtp774187871-55537) [n:127.0.0.1:38955_btbk    x:control_collection_shard1_replica_n1 ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 2484581 INFO  (qtp774187871-55537) [n:127.0.0.1:38955_btbk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.4.0
   [junit4]   2> 2484595 INFO  (qtp774187871-55537) [n:127.0.0.1:38955_btbk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
   [junit4]   2> 2484690 INFO  (qtp774187871-55537) [n:127.0.0.1:38955_btbk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 2484703 INFO  (qtp774187871-55537) [n:127.0.0.1:38955_btbk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
   [junit4]   2> 2484703 INFO  (qtp774187871-55537) [n:127.0.0.1:38955_btbk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2b3715f0
   [junit4]   2> 2484703 INFO  (qtp774187871-55537) [n:127.0.0.1:38955_btbk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_6537596EC6089843-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_6537596EC6089843-001/control-001/cores/control_collection_shard1_replica_n1/data/]
   [junit4]   2> 2484705 INFO  (qtp774187871-55537) [n:127.0.0.1:38955_btbk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=22, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7255663729025847]
   [junit4]   2> 2484713 WARN  (qtp774187871-55537) [n:127.0.0.1:38955_btbk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 2484745 INFO  (qtp774187871-55537) [n:127.0.0.1:38955_btbk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2484745 INFO  (qtp774187871-55537) [n:127.0.0.1:38955_btbk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2484746 INFO  (qtp774187871-55537) [n:127.0.0.1:38955_btbk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2484746 INFO  (qtp774187871-55537) [n:127.0.0.1:38955_btbk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2484747 INFO  (qtp774187871-55537) [n:127.0.0.1:38955_btbk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@66124f3c
   [junit4]   2> 2484747 INFO  (qtp774187871-55537) [n:127.0.0.1:38955_btbk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.s.SolrIndexSearcher Opening [Searcher@28456653[control_collection_shard1_replica_n1] main]
   [junit4]   2> 2484748 INFO  (qtp774187871-55537) [n:127.0.0.1:38955_btbk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 2484748 INFO  (qtp774187871-55537) [n:127.0.0.1:38955_btbk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2484748 INFO  (qtp774187871-55537) [n:127.0.0.1:38955_btbk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 2484748 INFO  (qtp774187871-55537) [n:127.0.0.1:38955_btbk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1651822357917990912
   [junit4]   2> 2484751 INFO  (searcherExecutor-11435-thread-1-processing-n:127.0.0.1:38955_btbk x:control_collection_shard1_replica_n1 c:control_collection s:shard1) [n:127.0.0.1:38955_btbk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@28456653[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2484752 INFO  (qtp774187871-55537) [n:127.0.0.1:38955_btbk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.ZkShardTerms Successful update of terms at /collections/control_collection/terms/shard1 to Terms{values={core_node2=0}, version=0}
   [junit4]   2> 2484752 INFO  (qtp774187871-55537) [n:127.0.0.1:38955_btbk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/control_collection/leaders/shard1
   [junit4]   2> 2484754 INFO  (qtp774187871-55537) [n:127.0.0.1:38955_btbk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 2484754 INFO  (qtp774187871-55537) [n:127.0.0.1:38955_btbk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 2484754 INFO  (qtp774187871-55537) [n:127.0.0.1:38955_btbk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:38955/btbk/control_collection_shard1_replica_n1/
   [junit4]   2> 2484754 INFO  (qtp774187871-55537) [n:127.0.0.1:38955_btbk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 2484754 INFO  (qtp774187871-55537) [n:127.0.0.1:38955_btbk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.SyncStrategy http://127.0.0.1:38955/btbk/control_collection_shard1_replica_n1/ has no replicas
   [junit4]   2> 2484754 INFO  (qtp774187871-55537) [n:127.0.0.1:38955_btbk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/control_collection/leaders/shard1/leader after winning as /collections/control_collection/leader_elect/shard1/election/72825816818712580-core_node2-n_0000000000
   [junit4]   2> 2484755 INFO  (qtp774187871-55537) [n:127.0.0.1:38955_btbk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:38955/btbk/control_collection_shard1_replica_n1/ shard1
   [junit4]   2> 2484856 INFO  (zkCallback-12061-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 2484856 INFO  (zkCallback-12061-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 2484857 INFO  (qtp774187871-55537) [n:127.0.0.1:38955_btbk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 2484858 INFO  (qtp774187871-55537) [n:127.0.0.1:38955_btbk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1288
   [junit4]   2> 2484860 INFO  (qtp774187871-55536) [n:127.0.0.1:38955_btbk     ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
   [junit4]   2> 2484964 INFO  (zkCallback-12061-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 2484964 INFO  (zkCallback-12061-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 2484964 INFO  (zkCallback-12061-thread-3) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 2484964 INFO  (qtp774187871-55536) [n:127.0.0.1:38955_btbk     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:38955_btbk&wt=javabin&version=2} status=0 QTime=1500
   [junit4]   2> 2484964 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.AbstractFullDistribZkTestBase Waiting to see 1 active replicas in collection: control_collection
   [junit4]   2> 2485071 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2485072 INFO  (zkConnectionManagerCallback-12077-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2485072 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2485073 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2485073 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:34263/solr ready
   [junit4]   2> 2485073 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 2485074 INFO  (qtp774187871-55537) [n:127.0.0.1:38955_btbk     ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2485076 INFO  (OverseerCollectionConfigSetProcessor-72825816818712580-127.0.0.1:38955_btbk-n_0000000000) [n:127.0.0.1:38955_btbk     ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 2485076 INFO  (OverseerThreadFactory-11430-thread-2-processing-n:127.0.0.1:38955_btbk) [n:127.0.0.1:38955_btbk     ] o.a.s.c.a.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 2485278 WARN  (OverseerThreadFactory-11430-thread-2-processing-n:127.0.0.1:38955_btbk) [n:127.0.0.1:38955_btbk     ] o.a.s.c.a.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
   [junit4]   2> 2485279 INFO  (qtp774187871-55537) [n:127.0.0.1:38955_btbk     ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
   [junit4]   2> 2485279 INFO  (qtp774187871-55537) [n:127.0.0.1:38955_btbk     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=205
   [junit4]   2> 2485280 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.SolrCloudTestCase active slice count: 2 expected:2
   [junit4]   2> 2485280 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.SolrCloudTestCase active replica count: 0 expected replica count: 0
   [junit4]   2> 2485280 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.SolrCloudTestCase active slice count: 2 expected:2
   [junit4]   2> 2485280 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.SolrCloudTestCase active replica count: 0 expected replica count: 0
   [junit4]   2> 2485280 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.SolrCloudTestCase active slice count: 2 expected:2
   [junit4]   2> 2485280 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.SolrCloudTestCase active replica count: 0 expected replica count: 0
   [junit4]   2> 2485280 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.AbstractFullDistribZkTestBase Creating jetty instances pullReplicaCount=0 numOtherReplicas=16
   [junit4]   2> 2485396 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_6537596EC6089843-001/shard-1-001 of type NRT
   [junit4]   2> 2485397 WARN  (closeThreadPool-12078-thread-1) [     ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 2485397 INFO  (closeThreadPool-12078-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 2485397 INFO  (closeThreadPool-12078-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 2485397 INFO  (closeThreadPool-12078-thread-1) [     ] o.e.j.s.Server jetty-9.4.19.v20190610; built: 2019-06-10T16:30:51.723Z; git: afcf563148970e98786327af5e07c261fda175d3; jvm 1.8.0_191-b12
   [junit4]   2> 2485398 INFO  (closeThreadPool-12078-thread-1) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 2485398 INFO  (closeThreadPool-12078-thread-1) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 2485398 INFO  (closeThreadPool-12078-thread-1) [     ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 2485398 INFO  (closeThreadPool-12078-thread-1) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@662811{/btbk,null,AVAILABLE}
   [junit4]   2> 2485398 INFO  (closeThreadPool-12078-thread-1) [     ] o.e.j.s.AbstractConnector Started ServerConnector@f350b9d{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:34419}
   [junit4]   2> 2485398 INFO  (closeThreadPool-12078-thread-1) [     ] o.e.j.s.Server Started @2485427ms
   [junit4]   2> 2485398 INFO  (closeThreadPool-12078-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_6537596EC6089843-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/btbk, hostPort=34419, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_6537596EC6089843-001/shard-1-001/cores}
   [junit4]   2> 2485398 ERROR (closeThreadPool-12078-thread-1) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2485398 INFO  (closeThreadPool-12078-thread-1) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 2485398 INFO  (closeThreadPool-12078-thread-1) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.4.0
   [junit4]   2> 2485398 INFO  (closeThreadPool-12078-thread-1) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2485398 INFO  (closeThreadPool-12078-thread-1) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2485399 INFO  (closeThreadPool-12078-thread-1) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-12-02T15:29:18.662Z
   [junit4]   2> 2485403 INFO  (closeThreadPool-12078-thread-1) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2485404 INFO  (zkConnectionManagerCallback-12080-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2485404 INFO  (closeThreadPool-12078-thread-1) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2485505 INFO  (closeThreadPool-12078-thread-1) [     ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 2485505 INFO  (closeThreadPool-12078-thread-1) [     ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_6537596EC6089843-001/shard-1-001/solr.xml
   [junit4]   2> 2485508 INFO  (closeThreadPool-12078-thread-1) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 2485508 INFO  (closeThreadPool-12078-thread-1) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 2485509 INFO  (closeThreadPool-12078-thread-1) [     ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@2b3715f0, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 2485511 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_6537596EC6089843-001/shard-2-001 of type NRT
   [junit4]   2> 2485512 WARN  (closeThreadPool-12078-thread-2) [     ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 2485512 INFO  (closeThreadPool-12078-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 2485512 INFO  (closeThreadPool-12078-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 2485512 INFO  (closeThreadPool-12078-thread-2) [     ] o.e.j.s.Server jetty-9.4.19.v20190610; built: 2019-06-10T16:30:51.723Z; git: afcf563148970e98786327af5e07c261fda175d3; jvm 1.8.0_191-b12
   [junit4]   2> 2485515 INFO  (closeThreadPool-12078-thread-2) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 2485515 INFO  (closeThreadPool-12078-thread-2) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 2485515 INFO  (closeThreadPool-12078-thread-2) [     ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 2485516 INFO  (closeThreadPool-12078-thread-2) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4d4d628f{/btbk,null,AVAILABLE}
   [junit4]   2> 2485516 INFO  (closeThreadPool-12078-thread-2) [     ] o.e.j.s.AbstractConnector Started ServerConnector@3dc8bb82{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:43371}
   [junit4]   2> 2485516 INFO  (closeThreadPool-12078-thread-2) [     ] o.e.j.s.Server Started @2485545ms
   [junit4]   2> 2485516 INFO  (closeThreadPool-12078-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_6537596EC6089843-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostContext=/btbk, hostPort=43371, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_6537596EC6089843-001/shard-2-001/cores}
   [junit4]   2> 2485516 ERROR (closeThreadPool-12078-thread-2) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2485516 INFO  (closeThreadPool-12078-thread-2) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 2485516 INFO  (closeThreadPool-12078-thread-2) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.4.0
   [junit4]   2> 2485516 INFO  (closeThreadPool-12078-thread-2) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2485516 INFO  (closeThreadPool-12078-thread-2) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2485516 INFO  (closeThreadPool-12078-thread-2) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-12-02T15:29:18.780Z
   [junit4]   2> 2485517 INFO  (closeThreadPool-12078-thread-2) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2485518 INFO  (zkConnectionManagerCallback-12083-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2485518 INFO  (closeThreadPool-12078-thread-2) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2485618 INFO  (closeThreadPool-12078-thread-2) [     ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 2485618 INFO  (closeThreadPool-12078-thread-2) [     ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_6537596EC6089843-001/shard-2-001/solr.xml
   [junit4]   2> 2485622 INFO  (closeThreadPool-12078-thread-2) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 2485622 INFO  (closeThreadPool-12078-thread-2) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 2485623 INFO  (closeThreadPool-12078-thread-2) [     ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@2b3715f0, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 2485627 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_6537596EC6089843-001/shard-3-001 of type NRT
   [junit4]   2> 2485643 WARN  (closeThreadPool-12078-thread-3) [     ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 2485643 INFO  (closeThreadPool-12078-thread-3) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 2485643 INFO  (closeThreadPool-12078-thread-3) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 2485643 INFO  (closeThreadPool-12078-thread-3) [     ] o.e.j.s.Server jetty-9.4.19.v20190610; built: 2019-06-10T16:30:51.723Z; git: afcf563148970e98786327af5e07c261fda175d3; jvm 1.8.0_191-b12
   [junit4]   2> 2485658 INFO  (closeThreadPool-12078-thread-1) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
   [junit4]   2> 2485663 WARN  (closeThreadPool-12078-thread-1) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@4a027a50[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2485663 WARN  (closeThreadPool-12078-thread-1) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@4a027a50[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2485665 WARN  (closeThreadPool-12078-thread-1) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@7d67b480[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2485665 WARN  (closeThreadPool-12078-thread-1) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@7d67b480[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2485666 INFO  (closeThreadPool-12078-thread-1) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34263/solr
   [junit4]   2> 2485674 INFO  (closeThreadPool-12078-thread-1) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2485674 INFO  (closeThreadPool-12078-thread-3) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 2485674 INFO  (closeThreadPool-12078-thread-3) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 2485674 INFO  (closeThreadPool-12078-thread-3) [     ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 2485674 INFO  (closeThreadPool-12078-thread-3) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@573c15e0{/btbk,null,AVAILABLE}
   [junit4]   2> 2485678 INFO  (zkConnectionManagerCallback-12090-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2485678 INFO  (closeThreadPool-12078-thread-1) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2485678 INFO  (closeThreadPool-12078-thread-3) [     ] o.e.j.s.AbstractConnector Started ServerConnector@6046da9b{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:43051}
   [junit4]   2> 2485678 INFO  (closeThreadPool-12078-thread-3) [     ] o.e.j.s.Server Started @2485707ms
   [junit4]   2> 2485678 INFO  (closeThreadPool-12078-thread-3) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_6537596EC6089843-001/tempDir-001/jetty3, solrconfig=solrconfig.xml, hostContext=/btbk, hostPort=43051, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_6537596EC6089843-001/shard-3-001/cores}
   [junit4]   2> 2485678 ERROR (closeThreadPool-12078-thread-3) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2485678 INFO  (closeThreadPool-12078-thread-3) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 2485678 INFO  (closeThreadPool-12078-thread-3) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.4.0
   [junit4]   2> 2485678 INFO  (closeThreadPool-12078-thread-3) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2485678 INFO  (closeThreadPool-12078-thread-3) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2485678 INFO  (closeThreadPool-12078-thread-3) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-12-02T15:29:18.942Z
   [junit4]   2> 2485679 INFO  (closeThreadPool-12078-thread-3) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2485679 INFO  (zkConnectionManagerCallback-12092-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2485679 INFO  (closeThreadPool-12078-thread-3) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2485754 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 4 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_6537596EC6089843-001/shard-4-001 of type NRT
   [junit4]   2> 2485754 WARN  (closeThreadPool-12078-thread-4) [     ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 2485754 INFO  (closeThreadPool-12078-thread-4) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 2485754 INFO  (closeThreadPool-12078-thread-4) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 2485754 INFO  (closeThreadPool-12078-thread-4) [     ] o.e.j.s.Server jetty-9.4.19.v20190610; built: 2019-06-10T16:30:51.723Z; git: afcf563148970e98786327af5e07c261fda175d3; jvm 1.8.0_191-b12
   [junit4]   2> 2485755 INFO  (closeThreadPool-12078-thread-4) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 2485755 INFO  (closeThreadPool-12078-thread-4) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 2485755 INFO  (closeThreadPool-12078-thread-4) [     ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 2485755 INFO  (closeThreadPool-12078-thread-4) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3da642ac{/btbk,null,AVAILABLE}
   [junit4]   2> 2485755 INFO  (closeThreadPool-12078-thread-4) [     ] o.e.j.s.AbstractConnector Started ServerConnector@5d41eaa8{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:39471}
   [junit4]   2> 2485755 INFO  (closeThreadPool-12078-thread-4) [     ] o.e.j.s.Server Started @2485784ms
   [junit4]   2> 2485755 INFO  (closeThreadPool-12078-thread-4) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_6537596EC6089843-001/tempDir-001/jetty4, solrconfig=solrconfig.xml, hostContext=/btbk, hostPort=39471, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_6537596EC6089843-001/shard-4-001/cores}
   [junit4]   2> 2485756 ERROR (closeThreadPool-12078-thread-4) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2485756 INFO  (closeThreadPool-12078-thread-4) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 2485756 INFO  (closeThreadPool-12078-thread-4) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.4.0
   [junit4]   2> 2485756 INFO  (closeThreadPool-12078-thread-4) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2485756 INFO  (closeThreadPool-12078-thread-4) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2485756 INFO  (closeThreadPool-12078-thread-4) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-12-02T15:29:19.020Z
   [junit4]   2> 2485759 INFO  (closeThreadPool-12078-thread-4) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2485759 INFO  (zkConnectionManagerCallback-12094-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2485759 INFO  (closeThreadPool-12078-thread-4) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2485766 INFO  (closeThreadPool-12078-thread-2) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
   [junit4]   2> 2485766 WARN  (closeThreadPool-12078-thread-2) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@4d85b65[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2485766 WARN  (closeThreadPool-12078-thread-2) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@4d85b65[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2485769 WARN  (closeThreadPool-12078-thread-2) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@5bd40494[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2485769 WARN  (closeThreadPool-12078-thread-2) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@5bd40494[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2485770 INFO  (closeThreadPool-12078-thread-2) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34263/solr
   [junit4]   2> 2485770 INFO  (closeThreadPool-12078-thread-2) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2485771 INFO  (zkConnectionManagerCallback-12100-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2485771 INFO  (closeThreadPool-12078-thread-2) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2485779 INFO  (closeThreadPool-12078-thread-1) [n:127.0.0.1:34419_btbk     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2485780 INFO  (zkConnectionManagerCallback-12102-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2485780 INFO  (closeThreadPool-12078-thread-1) [n:127.0.0.1:34419_btbk     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2485784 INFO  (closeThreadPool-12078-thread-1) [n:127.0.0.1:34419_btbk     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2485786 INFO  (closeThreadPool-12078-thread-1) [n:127.0.0.1:34419_btbk     ] o.a.s.c.ZkController Publish node=127.0.0.1:34419_btbk as DOWN
   [junit4]   2> 2485786 INFO  (closeThreadPool-12078-thread-1) [n:127.0.0.1:34419_btbk     ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 2485786 INFO  (closeThreadPool-12078-thread-1) [n:127.0.0.1:34419_btbk     ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:34419_btbk
   [junit4]   2> 2485787 INFO  (zkCallback-12061-thread-3) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2485787 INFO  (zkCallback-12076-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2485791 INFO  (zkCallback-12101-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2485799 INFO  (closeThreadPool-12078-thread-1) [n:127.0.0.1:34419_btbk     ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 2485811 INFO  (closeThreadPool-12078-thread-1) [n:127.0.0.1:34419_btbk     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2b3715f0
   [junit4]   2> 2485817 INFO  (closeThreadPool-12078-thread-1) [n:127.0.0.1:34419_btbk     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2b3715f0
   [junit4]   2> 2485817 INFO  (closeThreadPool-12078-thread-1) [n:127.0.0.1:34419_btbk     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2b3715f0
   [junit4]   2> 2485818 INFO  (closeThreadPool-12078-thread-1) [n:127.0.0.1:34419_btbk     ] o.a.s.p.PackageLoader /packages.json updated to version -1
   [junit4]   2> 2485819 INFO  (closeThreadPool-12078-thread-1) [n:127.0.0.1:34419_btbk     ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_6537596EC6089843-001/shard-1-001/cores
   [junit4]   2> 2485851 INFO  (closeThreadPool-12078-thread-1) [     ] o.a.s.c.AbstractFullDistribZkTestBase waitForLiveNode: 127.0.0.1:34419_btbk
   [junit4]   2> 2485879 INFO  (closeThreadPool-12078-thread-2) [n:127.0.0.1:43371_btbk     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2485880 INFO  (zkConnectionManagerCallback-12108-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2485880 INFO  (closeThreadPool-12078-thread-2) [n:127.0.0.1:43371_btbk     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2485883 INFO  (closeThreadPool-12078-thread-2) [n:127.0.0.1:43371_btbk     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2485885 INFO  (closeThreadPool-12078-thread-2) [n:127.0.0.1:43371_btbk     ] o.a.s.c.ZkController Publish node=127.0.0.1:43371_btbk as DOWN
   [junit4]   2> 2485886 INFO  (closeThreadPool-12078-thread-2) [n:127.0.0.1:43371_btbk     ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 2485886 INFO  (closeThreadPool-12078-thread-2) [n:127.0.0.1:43371_btbk     ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43371_btbk
   [junit4]   2> 2485886 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 5 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_6537596EC6089843-001/shard-5-001 of type NRT
   [junit4]   2> 2485886 WARN  (closeThreadPool-12078-thread-1) [     ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 2485886 INFO  (zkCallback-12061-thread-3) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2485886 INFO  (zkCallback-12101-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2485886 INFO  (zkCallback-12076-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2485887 INFO  (closeThreadPool-12078-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 2485887 INFO  (closeThreadPool-12078-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 2485887 INFO  (closeThreadPool-12078-thread-1) [     ] o.e.j.s.Server jetty-9.4.19.v20190610; built: 2019-06-10T16:30:51.723Z; git: afcf563148970e98786327af5e07c261fda175d3; jvm 1.8.0_191-b12
   [junit4]   2> 2485891 INFO  (zkCallback-12107-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2485894 INFO  (closeThreadPool-12078-thread-1) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 2485894 INFO  (closeThreadPool-12078-thread-1) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 2485894 INFO  (closeThreadPool-12078-thread-1) [     ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 2485895 INFO  (closeThreadPool-12078-thread-1) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@603e009c{/btbk,null,AVAILABLE}
   [junit4]   2> 2485895 INFO  (closeThreadPool-12078-thread-1) [     ] o.e.j.s.AbstractConnector Started ServerConnector@50ae4d21{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:43669}
   [junit4]   2> 2485895 INFO  (closeThreadPool-12078-thread-1) [     ] o.e.j.s.Server Started @2485924ms
   [junit4]   2> 2485895 INFO  (closeThreadPool-12078-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_6537596EC6089843-001/tempDir-001/jetty5, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/btbk, hostPort=43669, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_6537596EC6089843-001/shard-5-001/cores}
   [junit4]   2> 2485895 ERROR (closeThreadPool-12078-thread-1) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2485896 INFO  (closeThreadPool-12078-thread-1) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 2485896 INFO  (closeThreadPool-12078-thread-1) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.4.0
   [junit4]   2> 2485896 INFO  (closeThreadPool-12078-thread-1) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2485896 INFO  (closeThreadPool-12078-thread-1) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2485896 INFO  (closeThreadPool-12078-thread-1) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-12-02T15:29:19.160Z
   [junit4]   2> 2485899 INFO  (closeThreadPool-12078-thread-1) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2485899 INFO  (zkConnectionManagerCallback-12113-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2485899 INFO  (closeThreadPool-12078-thread-1) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2485903 INFO  (closeThreadPool-12078-thread-2) [n:127.0.0.1:43371_btbk     ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 2485914 INFO  (closeThreadPool-12078-thread-2) [n:127.0.0.1:43371_btbk     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2b3715f0
   [junit4]   2> 2485921 INFO  (closeThreadPool-12078-thread-2) [n:127.0.0.1:43371_btbk     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2b3715f0
   [junit4]   2> 2485921 INFO  (closeThreadPool-12078-thread-2) [n:127.0.0.1:43371_btbk     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2b3715f0
   [junit4]   2> 2485979 INFO  (closeThreadPool-12078-thread-4) [     ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 2485979 INFO  (closeThreadPool-12078-thread-4) [     ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_6537596EC6089843-001/shard-4-001/solr.xml
   [junit4]   2> 2485982 INFO  (closeThreadPool-12078-thread-4) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 2485982 INFO  (closeThreadPool-12078-thread-4) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 2485983 INFO  (closeThreadPool-12078-thread-4) [     ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@2b3715f0, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 2486003 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 6 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_6537596EC6089843-001/shard-6-001 of type NRT
   [junit4]   2> 2486004 INFO  (closeThreadPool-12078-thread-1) [     ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 2486004 INFO  (closeThreadPool-12078-thread-1) [     ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_6537596EC6089843-001/shard-5-001/solr.xml
   [junit4]   2> 2486004 WARN  (closeThreadPool-12078-thread-5) [     ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 2486004 INFO  (closeThreadPool-12078-thread-5) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 2486004 INFO  (closeThreadPool-12078-thread-5) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 2486004 INFO  (closeThreadPool-12078-thread-5) [     ] o.e.j.s.Server jetty-9.4.19.v20190610; built: 2019-06-10T16:30:51.723Z; git: afcf563148970e98786327af5e07c261fda175d3; jvm 1.8.0_191-b12
   [junit4]   2> 2486007 INFO  (closeThreadPool-12078-thread-1) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 2486007 INFO  (closeThreadPool-12078-thread-1) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 2486007 INFO  (closeThreadPool-12078-thread-2) [n:127.0.0.1:43371_btbk     ] o.a.s.p.PackageLoader /packages.json updated to version -1
   [junit4]   2> 2486008 INFO  (closeThreadPool-12078-thread-2) [n:127.0.0.1:43371_btbk     ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_6537596EC6089843-001/shard-2-001/cores
   [junit4]   2> 2486009 INFO  (closeThreadPool-12078-thread-1) [     ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@2b3715f0, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 2486015 INFO  (closeThreadPool-12078-thread-5) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 2486015 INFO  (closeThreadPool-12078-thread-5) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 2486015 INFO  (closeThreadPool-12078-thread-5) [     ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 2486016 INFO  (closeThreadPool-12078-thread-5) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@113125f{/btbk,null,AVAILABLE}
   [junit4]   2> 2486017 INFO  (closeThreadPool-12078-thread-5) [     ] o.e.j.s.AbstractConnector Started ServerConnector@14d3ec53{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:41409}
   [junit4]   2> 2486017 INFO  (closeThreadPool-12078-thread-5) [     ] o.e.j.s.Server Started @2486046ms
   [junit4]   2> 2486017 INFO  (closeThreadPool-12078-thread-5) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_6537596EC6089843-001/tempDir-001/jetty6, solrconfig=solrconfig.xml, hostContext=/btbk, hostPort=41409, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_6537596EC6089843-001/shard-6-001/cores}
   [junit4]   2> 2486017 ERROR (closeThreadPool-12078-thread-5) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2486017 INFO  (closeThreadPool-12078-thread-5) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 2486017 INFO  (closeThreadPool-12078-thread-5) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.4.0
   [junit4]   2> 2486017 INFO  (closeThreadPool-12078-thread-5) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2486017 INFO  (closeThreadPool-12078-thread-5) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2486017 INFO  (closeThreadPool-12078-thread-5) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-12-02T15:29:19.281Z
   [junit4]   2> 2486031 INFO  (closeThreadPool-12078-thread-5) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2486034 INFO  (zkConnectionManagerCallback-12118-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2486034 INFO  (closeThreadPool-12078-thread-5) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2486044 INFO  (closeThreadPool-12078-thread-2) [     ] o.a.s.c.AbstractFullDistribZkTestBase waitForLiveNode: 127.0.0.1:43371_btbk
   [junit4]   2> 2486067 INFO  (closeThreadPool-12078-thread-4) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
   [junit4]   2> 2486068 WARN  (closeThreadPool-12078-thread-4) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@535a807a[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2486068 WARN  (closeThreadPool-12078-thread-4) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@535a807a[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2486079 INFO  (closeThreadPool-12078-thread-3) [     ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 2486079 INFO  (closeThreadPool-12078-thread-3) [     ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_6537596EC6089843-001/shard-3-001/solr.xml
   [junit4]   2> 2486082 INFO  (closeThreadPool-12078-thread-3) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 2486082 INFO  (closeThreadPool-12078-thread-3) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 2486083 INFO  (closeThreadPool-12078-thread-3) [     ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@2b3715f0, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 2486087 WARN  (closeThreadPool-12078-thread-4) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@5c233cd4[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2486087 WARN  (closeThreadPool-12078-thread-4) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@5c233cd4[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2486088 INFO  (closeThreadPool-12078-thread-4) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34263/solr
   [junit4]   2> 2486102 INFO  (closeThreadPool-12078-thread-4) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2486112 INFO  (zkConnectionManagerCallback-12125-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2486112 INFO  (closeThreadPool-12078-thread-4) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2486134 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 7 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_6537596EC6089843-001/shard-7-001 of type NRT
   [junit4]   2> 2486134 WARN  (closeThreadPool-12078-thread-2) [     ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 2486134 INFO  (closeThreadPool-12078-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 2486134 INFO  (closeThreadPool-12078-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 2486134 INFO  (closeThreadPool-12078-thread-2) [     ] o.e.j.s.Server jetty-9.4.19.v20190610; built: 2019-06-10T16:30:51.723Z; git: afcf563148970e98786327af5e07c261fda175d3; jvm 1.8.0_191-b12
   [junit4]   2> 2486145 INFO  (closeThreadPool-12078-thread-2) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 2486145 INFO  (closeThreadPool-12078-thread-2) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 2486145 INFO  (closeThreadPool-12078-thread-2) [     ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 2486146 INFO  (closeThreadPool-12078-thread-2) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@24cb770{/btbk,null,AVAILABLE}
   [junit4]   2> 2486146 INFO  (closeThreadPool-12078-thread-2) [     ] o.e.j.s.AbstractConnector Started ServerConnector@5b1cf802{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:45499}
   [junit4]   2> 2486146 INFO  (closeThreadPool-12078-thread-2) [     ] o.e.j.s.Server Started @2486175ms
   [junit4]   2> 2486146 INFO  (closeThreadPool-12078-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_6537596EC6089843-001/tempDir-001/jetty7, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/btbk, hostPort=45499, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_6537596EC6089843-001/shard-7-001/cores}
   [junit4]   2> 2486146 ERROR (closeThreadPool-12078-thread-2) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2486146 INFO  (closeThreadPool-12078-thread-2) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 2486146 INFO  (closeThreadPool-12078-thread-2) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.4.0
   [junit4]   2> 2486146 INFO  (closeThreadPool-12078-thread-2) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2486146 INFO  (closeThreadPool-12078-thread-2) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2486146 INFO  (closeThreadPool-12078-thread-2) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-12-02T15:29:19.410Z
   [junit4]   2> 2486147 INFO  (closeThreadPool-12078-thread-2) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2486148 INFO  (closeThreadPool-12078-thread-1) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
   [junit4]   2> 2486148 INFO  (zkConnectionManagerCallback-12127-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2486148 INFO  (closeThreadPool-12078-thread-2) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2486149 WARN  (closeThreadPool-12078-thread-1) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@34326e84[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2486149 WARN  (closeThreadPool-12078-thread-1) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@34326e84[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2486156 WARN  (closeThreadPool-12078-thread-1) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@346416aa[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2486156 WARN  (closeThreadPool-12078-thread-1) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@346416aa[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2486157 INFO  (closeThreadPool-12078-thread-1) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34263/solr
   [junit4]   2> 2486159 INFO  (closeThreadPool-12078-thread-1) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2486163 INFO  (zkConnectionManagerCallback-12133-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2486163 INFO  (closeThreadPool-12078-thread-1) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2486179 INFO  (closeThreadPool-12078-thread-5) [     ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 2486179 INFO  (closeThreadPool-12078-thread-5) [     ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_6537596EC6089843-001/shard-6-001/solr.xml
   [junit4]   2> 2486195 INFO  (closeThreadPool-12078-thread-5) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 2486195 INFO  (closeThreadPool-12078-thread-5) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 2486196 INFO  (closeThreadPool-12078-thread-5) [     ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@2b3715f0, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 2486200 INFO  (closeThreadPool-12078-thread-3) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
   [junit4]   2> 2486200 WARN  (closeThreadPool-12078-thread-3) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@d2651e6[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2486200 WARN  (closeThreadPool-12078-thread-3) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@d2651e6[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2486202 WARN  (closeThreadPool-12078-thread-3) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@ca964e3[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2486202 WARN  (closeThreadPool-12078-thread-3) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@ca964e3[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2486203 INFO  (closeThreadPool-12078-thread-3) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34263/solr
   [junit4]   2> 2486204 INFO  (closeThreadPool-12078-thread-3) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2486204 INFO  (zkConnectionManagerCallback-12140-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2486204 INFO  (closeThreadPool-12078-thread-3) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2486272 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 8 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_6537596EC6089843-001/shard-8-001 of type NRT
   [junit4]   2> 2486272 WARN  (closeThreadPool-12078-thread-6) [     ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 2486272 INFO  (closeThreadPool-12078-thread-6) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 2486272 INFO  (closeThreadPool-12078-thread-6) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 2486273 INFO  (closeThreadPool-12078-thread-6) [     ] o.e.j.s.Server jetty-9.4.19.v20190610; built: 2019-06-10T16:30:51.723Z; git: afcf563148970e98786327af5e07c261fda175d3; jvm 1.8.0_191-b12
   [junit4]   2> 2486273 INFO  (closeThreadPool-12078-thread-6) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 2486273 INFO  (closeThreadPool-12078-thread-6) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 2486273 INFO  (closeThreadPool-12078-thread-6) [     ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 2486273 INFO  (closeThreadPool-12078-thread-6) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1b04fa37{/btbk,null,AVAILABLE}
   [junit4]   2> 2486274 INFO  (closeThreadPool-12078-thread-6) [     ] o.e.j.s.AbstractConnector Started ServerConnector@30c86acd{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:33467}
   [junit4]   2> 2486274 INFO  (closeThreadPool-12078-thread-6) [     ] o.e.j.s.Server Started @2486303ms
   [junit4]   2> 2486274 INFO  (closeThreadPool-12078-thread-6) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_6537596EC6089843-001/tempDir-001/jetty8, solrconfig=solrconfig.xml, hostContext=/btbk, hostPort=33467, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_6537596EC6089843-001/shard-8-001/cores}
   [junit4]   2> 2486274 ERROR (closeThreadPool-12

[...truncated too long message...]

ST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:34263
   [junit4]   2> 2626413 INFO  (TEST-RollingRestartTest.test-seed#[6537596EC6089843]) [     ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 34263
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RollingRestartTest -Dtests.method=test -Dtests.seed=6537596EC6089843 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/test-data/enwiki.random.lines.txt -Dtests.locale=sq -Dtests.timezone=Atlantic/Madeira -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR    144s J2 | RollingRestartTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: Timeout occurred while waiting response from server at: http://127.0.0.1:39471/btbk
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([6537596EC6089843:ED6366B468F4F5BB]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:676)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:265)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:368)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:296)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.sendRequest(BaseCloudSolrClient.java:1127)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.requestWithRetryOnStaleState(BaseCloudSolrClient.java:896)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.request(BaseCloudSolrClient.java:828)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:228)
   [junit4]    > 	at org.apache.solr.cloud.RollingRestartTest.restartWithRolesTest(RollingRestartTest.java:74)
   [junit4]    > 	at org.apache.solr.cloud.RollingRestartTest.test(RollingRestartTest.java:53)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1082)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1054)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > Caused by: java.net.SocketTimeoutException: Read timed out
   [junit4]    > 	at java.net.SocketInputStream.socketRead0(Native Method)
   [junit4]    > 	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
   [junit4]    > 	at java.net.SocketInputStream.read(SocketInputStream.java:171)
   [junit4]    > 	at java.net.SocketInputStream.read(SocketInputStream.java:141)
   [junit4]    > 	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
   [junit4]    > 	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
   [junit4]    > 	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
   [junit4]    > 	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
   [junit4]    > 	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
   [junit4]    > 	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
   [junit4]    > 	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
   [junit4]    > 	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
   [junit4]    > 	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
   [junit4]    > 	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:564)
   [junit4]    > 	... 52 more
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_6537596EC6089843-001
   [junit4]   2> NOTE: test params are: codec=SimpleText, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@5d18a65e), locale=sq, timezone=Atlantic/Madeira
   [junit4]   2> NOTE: Linux 4.15.0-54-generic amd64/Oracle Corporation 1.8.0_191 (64-bit)/cpus=4,threads=1,free=235940872,total=533725184
   [junit4]   2> NOTE: All tests run in this JVM: [TestFacetMethods, RulesTest, TestRandomFlRTGCloud, TestTolerantSearch, TestSimPolicyCloud, TestDynamicLoadingUrl, TestCoreBackup, TestSnapshotCloudManager, TestEmbeddedSolrServerConstructors, SystemInfoHandlerTest, TestExportTool, DaemonStreamApiTest, ResourceLoaderTest, TestQueryTypes, TestUtilizeNode, MergeStrategyTest, TestFieldCollectionResource, TestFieldCache, SyncSliceTest, TestSolrCloudSnapshots, SecurityConfHandlerTest, SolrIndexSplitterTest, TestDynamicFieldNamesIndexCorrectly, ShardsWhitelistTest, TestCaffeineCache, HdfsChaosMonkeyNothingIsSafeTest, TestSolrCLIRunExample, TriggerIntegrationTest, FileUtilsTest, SolrCoreMetricManagerTest, TestCloudPhrasesIdentificationComponent, TestSolrCloudWithDelegationTokens, TestExactStatsCache, HttpSolrCallGetCoreTest, UtilsToolTest, SolrShardReporterTest, TestMinMaxOnMultiValuedField, SolrCloudExampleTest, TestCopyFieldCollectionResource, TestRealTimeGet, TestSimpleTrackingShardHandler, TestLogWatcher, TestPushWriter, TestJsonFacetRefinement, FileBasedSpellCheckerTest, FullSolrCloudDistribCmdsTest, TestMaxScoreQueryParser, CloudMLTQParserTest, TestManagedSynonymGraphFilterFactory, CdcrRequestHandlerTest, TestCollectionsAPIViaSolrCloudCluster, PingRequestHandlerTest, TestZkAclsWithHadoopAuth, BigEndianAscendingWordDeserializerTest, TestCSVResponseWriter, TestMultiWordSynonyms, TestFileDictionaryLookup, CoreAdminOperationTest, HdfsLockFactoryTest, TestSolrQueryParser, ZkControllerTest, HdfsSyncSliceTest, BasicDistributedZkTest, NumericFieldsTest, DocValuesNotIndexedTest, TestTlogReplica, ClassificationUpdateProcessorIntegrationTest, TestClusterStateMutator, TestPullReplica, TestBinaryField, TestSolrXml, DocExpirationUpdateProcessorFactoryTest, TestCustomStream, CollectionStateFormat2Test, TestRandomCollapseQParserPlugin, HdfsUnloadDistributedZkTest, TestCloudConsistency, IgnoreLargeDocumentProcessorFactoryTest, TestDefaultStatsCache, SuggesterTSTTest, TestStressUserVersions, QueryResultKeyTest, TestLazyCores, TestBlobHandler, ConvertedLegacyTest, HLLUtilTest, EnumFieldTest, TestInfoStreamLogging, PKIAuthenticationIntegrationTest, DefaultValueUpdateProcessorTest, TestUniqueKeyFieldResource, NodeLostTriggerTest, CdcrBootstrapTest, TestQuerySenderListener, ConfigureRecoveryStrategyTest, HDFSCollectionsAPITest, TermVectorComponentTest, TestLegacyFieldCache, CustomCollectionTest, TestSlowCompositeReaderWrapper, TestScoreJoinQPScore, TestConfigSetsAPIZkFailure, TestSimLargeCluster, HdfsRecoverLeaseTest, OutOfBoxZkACLAndCredentialsProvidersTest, SearchRateTriggerIntegrationTest, ExitableDirectoryReaderTest, XmlUpdateRequestHandlerTest, TestDocBasedVersionConstraints, TestRestoreCore, TestCloudSchemaless, TestJoin, SolrCoreTest, AliasIntegrationTest, ParsingFieldUpdateProcessorsTest, MetricUtilsTest, TestSchemaField, TestComplexPhraseQParserPlugin, ReindexCollectionTest, TestHalfAndHalfDocValues, PeerSyncWithLeaderTest, TestSchemaSimilarityResource, IndexSchemaTest, LeaderFailureAfterFreshStartTest, DistributedFacetPivotSmallAdvancedTest, TestReload, RequiredFieldsTest, OverseerModifyCollectionTest, CurrencyRangeFacetCloudTest, SharedFSAutoReplicaFailoverTest, CollectionsAPIAsyncDistributedZkTest, TokenizerChainTest, TestHdfsUpdateLog, DistributedIntervalFacetingTest, TestExpandComponent, TestSolrConfigHandlerCloud, TestSimComputePlanAction, TestLFUCache, BasicAuthStandaloneTest, UUIDUpdateProcessorFallbackTest, TestDocumentBuilder, TestTrackingShardHandlerFactory, CurrencyFieldTypeTest, PeerSyncWithBufferUpdatesTest, TestLeaderElectionWithEmptyReplica, ClusterStateTest, ClusterStateUpdateTest, CollectionPropsTest, ConfigSetsAPITest, CreateRoutedAliasTest, DeleteStatusTest, ForceLeaderTest, HttpPartitionTest, LeaderElectionContextKeyTest, LeaderElectionIntegrationTest, LeaderFailoverAfterPartitionTest, LegacyCloudClusterPropTest, MigrateRouteKeyTest, MultiSolrCloudTestCaseTest, MultiThreadedOCPTest, NestedShardedAtomicUpdateTest, OverseerStatusTest, OverseerTaskQueueTest, PackageManagerCLITest, RecoveryAfterSoftCommitTest, RecoveryZkTest, RemoteQueryErrorTest, ReplaceNodeNoTargetTest, ReplaceNodeTest, ReplicationFactorTest, RollingRestartTest]
   [junit4] Completed [444/887 (1!)] on J2 in 143.76s, 1 test, 1 error <<< FAILURES!

[...truncated 54331 lines...]

[JENKINS] Lucene-Solr-NightlyTests-8.x - Build # 300 - Still Unstable

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-8.x/300/

1 tests failed.
FAILED:  org.apache.solr.cloud.RollingRestartTest.test

Error Message:
Timeout occurred while waiting response from server at: http://127.0.0.1:44885/lf_ei/uk

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occurred while waiting response from server at: http://127.0.0.1:44885/lf_ei/uk
	at __randomizedtesting.SeedInfo.seed([B11268ACB6631A6B:39465776189F7793]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:676)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:265)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:368)
	at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:296)
	at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.sendRequest(BaseCloudSolrClient.java:1143)
	at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.requestWithRetryOnStaleState(BaseCloudSolrClient.java:906)
	at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.request(BaseCloudSolrClient.java:838)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:228)
	at org.apache.solr.cloud.RollingRestartTest.restartWithRolesTest(RollingRestartTest.java:74)
	at org.apache.solr.cloud.RollingRestartTest.test(RollingRestartTest.java:53)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1082)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1054)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:171)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:280)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:157)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:564)
	... 52 more




Build Log:
[...truncated 15236 lines...]
   [junit4] Suite: org.apache.solr.cloud.RollingRestartTest
   [junit4]   2> 2382199 INFO  (SUITE-RollingRestartTest-seed#[B11268ACB6631A6B]-worker) [     ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 2382199 INFO  (SUITE-RollingRestartTest-seed#[B11268ACB6631A6B]-worker) [     ] o.a.s.SolrTestCaseJ4 Created dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_B11268ACB6631A6B-001/data-dir-182-001
   [junit4]   2> 2382200 WARN  (SUITE-RollingRestartTest-seed#[B11268ACB6631A6B]-worker) [     ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=10 numCloses=10
   [junit4]   2> 2382200 INFO  (SUITE-RollingRestartTest-seed#[B11268ACB6631A6B]-worker) [     ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 2382202 INFO  (SUITE-RollingRestartTest-seed#[B11268ACB6631A6B]-worker) [     ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 2382202 INFO  (SUITE-RollingRestartTest-seed#[B11268ACB6631A6B]-worker) [     ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /lf_ei/uk
   [junit4]   2> 2382206 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 2382207 INFO  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 2382207 INFO  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 2382307 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.ZkTestServer start zk server on port:39977
   [junit4]   2> 2382307 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.ZkTestServer waitForServerUp: 127.0.0.1:39977
   [junit4]   2> 2382307 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:39977
   [junit4]   2> 2382307 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 39977
   [junit4]   2> 2382314 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2382322 INFO  (zkConnectionManagerCallback-12278-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2382322 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2382327 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2382328 INFO  (zkConnectionManagerCallback-12280-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2382328 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2382329 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 2382329 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 2382330 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 2382331 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 2382331 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 2382331 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 2382332 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 2382332 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 2382333 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 2382333 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 2382334 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 2382334 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
   [junit4]   2> 2382410 WARN  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
   [junit4]   2> 2382410 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 2382410 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 2382410 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z; git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 1.8.0_191-b12
   [junit4]   2> 2382414 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 2382414 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 2382415 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 2382431 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@79d0e458{/lf_ei/uk,null,AVAILABLE}
   [junit4]   2> 2382431 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.e.j.s.AbstractConnector Started ServerConnector@67d6122d{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:46093}
   [junit4]   2> 2382431 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.e.j.s.Server Started @2382457ms
   [junit4]   2> 2382431 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_B11268ACB6631A6B-001/tempDir-001/control/data, replicaType=NRT, hostContext=/lf_ei/uk, hostPort=46093, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_B11268ACB6631A6B-001/control-001/cores}
   [junit4]   2> 2382431 ERROR (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2382431 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 2382431 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.5.0
   [junit4]   2> 2382431 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2382431 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2382431 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-12-19T15:45:01.332Z
   [junit4]   2> 2382433 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2382434 INFO  (zkConnectionManagerCallback-12282-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2382434 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2382534 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 2382535 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_B11268ACB6631A6B-001/control-001/solr.xml
   [junit4]   2> 2382538 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 2382538 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 2382539 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@345572b9, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 2382596 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
   [junit4]   2> 2382597 WARN  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@28207e9c[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2382597 WARN  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@28207e9c[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2382599 WARN  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@4b4691ec[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2382599 WARN  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@4b4691ec[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2382600 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39977/solr
   [junit4]   2> 2382601 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2382602 INFO  (zkConnectionManagerCallback-12289-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2382602 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2382704 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [n:127.0.0.1:46093_lf_ei%2Fuk     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2382705 INFO  (zkConnectionManagerCallback-12291-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2382705 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [n:127.0.0.1:46093_lf_ei%2Fuk     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2382768 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [n:127.0.0.1:46093_lf_ei%2Fuk     ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:46093_lf_ei%2Fuk
   [junit4]   2> 2382769 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [n:127.0.0.1:46093_lf_ei%2Fuk     ] o.a.s.c.Overseer Overseer (id=72922138026311684-127.0.0.1:46093_lf_ei%2Fuk-n_0000000000) starting
   [junit4]   2> 2382782 INFO  (OverseerStateUpdate-72922138026311684-127.0.0.1:46093_lf_ei%2Fuk-n_0000000000) [n:127.0.0.1:46093_lf_ei%2Fuk     ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:46093_lf_ei%2Fuk
   [junit4]   2> 2382784 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [n:127.0.0.1:46093_lf_ei%2Fuk     ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:46093_lf_ei%2Fuk
   [junit4]   2> 2382786 WARN  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [n:127.0.0.1:46093_lf_ei%2Fuk     ] o.a.s.c.CoreContainer Not all security plugins configured!  authentication=disabled authorization=disabled.  Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external.  See https://s.apache.org/solrsecurity for more info
   [junit4]   2> 2382790 INFO  (zkCallback-12290-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2382800 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [n:127.0.0.1:46093_lf_ei%2Fuk     ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 2382814 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [n:127.0.0.1:46093_lf_ei%2Fuk     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@345572b9
   [junit4]   2> 2382820 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [n:127.0.0.1:46093_lf_ei%2Fuk     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@345572b9
   [junit4]   2> 2382820 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [n:127.0.0.1:46093_lf_ei%2Fuk     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@345572b9
   [junit4]   2> 2382821 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [n:127.0.0.1:46093_lf_ei%2Fuk     ] o.a.s.p.PackageLoader /packages.json updated to version -1
   [junit4]   2> 2382822 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [n:127.0.0.1:46093_lf_ei%2Fuk     ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_B11268ACB6631A6B-001/control-001/cores
   [junit4]   2> 2382838 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2382839 INFO  (zkConnectionManagerCallback-12300-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2382840 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2382841 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2382842 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:39977/solr ready
   [junit4]   2> 2382843 INFO  (qtp673033104-44061) [n:127.0.0.1:46093_lf_ei%2Fuk     ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:46093_lf_ei%252Fuk&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2382847 INFO  (OverseerThreadFactory-11961-thread-1-processing-n:127.0.0.1:46093_lf_ei%2Fuk) [n:127.0.0.1:46093_lf_ei%2Fuk     ] o.a.s.c.a.c.CreateCollectionCmd Create collection control_collection
   [junit4]   2> 2382953 INFO  (qtp673033104-44063) [n:127.0.0.1:46093_lf_ei%2Fuk    x:control_collection_shard1_replica_n1 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 2382953 INFO  (qtp673033104-44063) [n:127.0.0.1:46093_lf_ei%2Fuk    x:control_collection_shard1_replica_n1 ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 2383970 INFO  (qtp673033104-44063) [n:127.0.0.1:46093_lf_ei%2Fuk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.5.0
   [junit4]   2> 2383984 INFO  (qtp673033104-44063) [n:127.0.0.1:46093_lf_ei%2Fuk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
   [junit4]   2> 2384082 INFO  (qtp673033104-44063) [n:127.0.0.1:46093_lf_ei%2Fuk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 2384097 INFO  (qtp673033104-44063) [n:127.0.0.1:46093_lf_ei%2Fuk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
   [junit4]   2> 2384097 INFO  (qtp673033104-44063) [n:127.0.0.1:46093_lf_ei%2Fuk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@345572b9
   [junit4]   2> 2384097 INFO  (qtp673033104-44063) [n:127.0.0.1:46093_lf_ei%2Fuk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_B11268ACB6631A6B-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_B11268ACB6631A6B-001/control-001/cores/control_collection_shard1_replica_n1/data/]
   [junit4]   2> 2384100 INFO  (qtp673033104-44063) [n:127.0.0.1:46093_lf_ei%2Fuk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=28, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=0.6748046875, noCFSRatio=0.0]
   [junit4]   2> 2384113 WARN  (qtp673033104-44063) [n:127.0.0.1:46093_lf_ei%2Fuk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 2384146 INFO  (qtp673033104-44063) [n:127.0.0.1:46093_lf_ei%2Fuk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2384146 INFO  (qtp673033104-44063) [n:127.0.0.1:46093_lf_ei%2Fuk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2384156 INFO  (qtp673033104-44063) [n:127.0.0.1:46093_lf_ei%2Fuk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2384156 INFO  (qtp673033104-44063) [n:127.0.0.1:46093_lf_ei%2Fuk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2384157 INFO  (qtp673033104-44063) [n:127.0.0.1:46093_lf_ei%2Fuk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=25, maxMergeAtOnceExplicit=30, maxMergedSegmentMB=23.4990234375, floorSegmentMB=0.205078125, forceMergeDeletesPctAllowed=14.379124592887102, segmentsPerTier=10.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0, deletesPctAllowed=41.082138169390724
   [junit4]   2> 2384157 INFO  (qtp673033104-44063) [n:127.0.0.1:46093_lf_ei%2Fuk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.s.SolrIndexSearcher Opening [Searcher@48d0fe55[control_collection_shard1_replica_n1] main]
   [junit4]   2> 2384158 INFO  (qtp673033104-44063) [n:127.0.0.1:46093_lf_ei%2Fuk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 2384159 INFO  (qtp673033104-44063) [n:127.0.0.1:46093_lf_ei%2Fuk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2384159 INFO  (qtp673033104-44063) [n:127.0.0.1:46093_lf_ei%2Fuk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 2384159 INFO  (qtp673033104-44063) [n:127.0.0.1:46093_lf_ei%2Fuk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1653363497301442560
   [junit4]   2> 2384161 INFO  (searcherExecutor-11966-thread-1-processing-n:127.0.0.1:46093_lf_ei%2Fuk x:control_collection_shard1_replica_n1 c:control_collection s:shard1) [n:127.0.0.1:46093_lf_ei%2Fuk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@48d0fe55[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2384163 INFO  (qtp673033104-44063) [n:127.0.0.1:46093_lf_ei%2Fuk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.ZkShardTerms Successful update of terms at /collections/control_collection/terms/shard1 to Terms{values={core_node2=0}, version=0}
   [junit4]   2> 2384163 INFO  (qtp673033104-44063) [n:127.0.0.1:46093_lf_ei%2Fuk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/control_collection/leaders/shard1
   [junit4]   2> 2384165 INFO  (qtp673033104-44063) [n:127.0.0.1:46093_lf_ei%2Fuk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 2384165 INFO  (qtp673033104-44063) [n:127.0.0.1:46093_lf_ei%2Fuk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 2384165 INFO  (qtp673033104-44063) [n:127.0.0.1:46093_lf_ei%2Fuk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:46093/lf_ei/uk/control_collection_shard1_replica_n1/
   [junit4]   2> 2384165 INFO  (qtp673033104-44063) [n:127.0.0.1:46093_lf_ei%2Fuk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 2384165 INFO  (qtp673033104-44063) [n:127.0.0.1:46093_lf_ei%2Fuk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.SyncStrategy http://127.0.0.1:46093/lf_ei/uk/control_collection_shard1_replica_n1/ has no replicas
   [junit4]   2> 2384165 INFO  (qtp673033104-44063) [n:127.0.0.1:46093_lf_ei%2Fuk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/control_collection/leaders/shard1/leader after winning as /collections/control_collection/leader_elect/shard1/election/72922138026311684-core_node2-n_0000000000
   [junit4]   2> 2384166 INFO  (qtp673033104-44063) [n:127.0.0.1:46093_lf_ei%2Fuk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:46093/lf_ei/uk/control_collection_shard1_replica_n1/ shard1
   [junit4]   2> 2384268 INFO  (zkCallback-12290-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 2384268 INFO  (zkCallback-12290-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 2384268 INFO  (qtp673033104-44063) [n:127.0.0.1:46093_lf_ei%2Fuk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 2384270 INFO  (qtp673033104-44063) [n:127.0.0.1:46093_lf_ei%2Fuk c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1317
   [junit4]   2> 2384272 INFO  (qtp673033104-44061) [n:127.0.0.1:46093_lf_ei%2Fuk     ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
   [junit4]   2> 2384370 INFO  (zkCallback-12290-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 2384370 INFO  (zkCallback-12290-thread-3) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 2384370 INFO  (zkCallback-12290-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 2384371 INFO  (qtp673033104-44061) [n:127.0.0.1:46093_lf_ei%2Fuk     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:46093_lf_ei%252Fuk&wt=javabin&version=2} status=0 QTime=1528
   [junit4]   2> 2384371 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.AbstractFullDistribZkTestBase Waiting to see 1 active replicas in collection: control_collection
   [junit4]   2> 2384475 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2384476 INFO  (zkConnectionManagerCallback-12306-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2384476 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2384477 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2384478 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:39977/solr ready
   [junit4]   2> 2384478 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 2384482 INFO  (qtp673033104-44063) [n:127.0.0.1:46093_lf_ei%2Fuk     ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2384484 INFO  (OverseerThreadFactory-11961-thread-2-processing-n:127.0.0.1:46093_lf_ei%2Fuk) [n:127.0.0.1:46093_lf_ei%2Fuk     ] o.a.s.c.a.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 2384484 INFO  (OverseerCollectionConfigSetProcessor-72922138026311684-127.0.0.1:46093_lf_ei%2Fuk-n_0000000000) [n:127.0.0.1:46093_lf_ei%2Fuk     ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 2384686 WARN  (OverseerThreadFactory-11961-thread-2-processing-n:127.0.0.1:46093_lf_ei%2Fuk) [n:127.0.0.1:46093_lf_ei%2Fuk     ] o.a.s.c.a.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
   [junit4]   2> 2384687 INFO  (qtp673033104-44063) [n:127.0.0.1:46093_lf_ei%2Fuk     ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
   [junit4]   2> 2384688 INFO  (qtp673033104-44063) [n:127.0.0.1:46093_lf_ei%2Fuk     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=206
   [junit4]   2> 2384689 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.SolrCloudTestCase active slice count: 2 expected:2
   [junit4]   2> 2384689 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.SolrCloudTestCase active replica count: 0 expected replica count: 0
   [junit4]   2> 2384689 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.SolrCloudTestCase active slice count: 2 expected:2
   [junit4]   2> 2384689 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.SolrCloudTestCase active replica count: 0 expected replica count: 0
   [junit4]   2> 2384689 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.SolrCloudTestCase active slice count: 2 expected:2
   [junit4]   2> 2384689 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.SolrCloudTestCase active replica count: 0 expected replica count: 0
   [junit4]   2> 2384689 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.AbstractFullDistribZkTestBase Creating jetty instances pullReplicaCount=0 numOtherReplicas=16
   [junit4]   2> 2384767 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_B11268ACB6631A6B-001/shard-1-001 of type NRT
   [junit4]   2> 2384768 WARN  (closeThreadPool-12307-thread-1) [     ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
   [junit4]   2> 2384768 INFO  (closeThreadPool-12307-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 2384768 INFO  (closeThreadPool-12307-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 2384768 INFO  (closeThreadPool-12307-thread-1) [     ] o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z; git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 1.8.0_191-b12
   [junit4]   2> 2384770 INFO  (closeThreadPool-12307-thread-1) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 2384770 INFO  (closeThreadPool-12307-thread-1) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 2384770 INFO  (closeThreadPool-12307-thread-1) [     ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 2384770 INFO  (closeThreadPool-12307-thread-1) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@328e5304{/lf_ei/uk,null,AVAILABLE}
   [junit4]   2> 2384771 INFO  (closeThreadPool-12307-thread-1) [     ] o.e.j.s.AbstractConnector Started ServerConnector@68829e19{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:39973}
   [junit4]   2> 2384771 INFO  (closeThreadPool-12307-thread-1) [     ] o.e.j.s.Server Started @2384797ms
   [junit4]   2> 2384771 INFO  (closeThreadPool-12307-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_B11268ACB6631A6B-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/lf_ei/uk, hostPort=39973, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_B11268ACB6631A6B-001/shard-1-001/cores}
   [junit4]   2> 2384771 ERROR (closeThreadPool-12307-thread-1) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2384771 INFO  (closeThreadPool-12307-thread-1) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 2384771 INFO  (closeThreadPool-12307-thread-1) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.5.0
   [junit4]   2> 2384771 INFO  (closeThreadPool-12307-thread-1) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2384771 INFO  (closeThreadPool-12307-thread-1) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2384771 INFO  (closeThreadPool-12307-thread-1) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-12-19T15:45:03.672Z
   [junit4]   2> 2384772 INFO  (closeThreadPool-12307-thread-1) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2384774 INFO  (zkConnectionManagerCallback-12309-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2384774 INFO  (closeThreadPool-12307-thread-1) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2384847 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_B11268ACB6631A6B-001/shard-2-001 of type NRT
   [junit4]   2> 2384848 WARN  (closeThreadPool-12307-thread-2) [     ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
   [junit4]   2> 2384848 INFO  (closeThreadPool-12307-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 2384848 INFO  (closeThreadPool-12307-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 2384848 INFO  (closeThreadPool-12307-thread-2) [     ] o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z; git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 1.8.0_191-b12
   [junit4]   2> 2384849 INFO  (closeThreadPool-12307-thread-2) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 2384849 INFO  (closeThreadPool-12307-thread-2) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 2384849 INFO  (closeThreadPool-12307-thread-2) [     ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 2384849 INFO  (closeThreadPool-12307-thread-2) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@663b2ea7{/lf_ei/uk,null,AVAILABLE}
   [junit4]   2> 2384849 INFO  (closeThreadPool-12307-thread-2) [     ] o.e.j.s.AbstractConnector Started ServerConnector@4375bb43{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:44885}
   [junit4]   2> 2384849 INFO  (closeThreadPool-12307-thread-2) [     ] o.e.j.s.Server Started @2384876ms
   [junit4]   2> 2384849 INFO  (closeThreadPool-12307-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_B11268ACB6631A6B-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostContext=/lf_ei/uk, hostPort=44885, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_B11268ACB6631A6B-001/shard-2-001/cores}
   [junit4]   2> 2384850 ERROR (closeThreadPool-12307-thread-2) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2384850 INFO  (closeThreadPool-12307-thread-2) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 2384850 INFO  (closeThreadPool-12307-thread-2) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.5.0
   [junit4]   2> 2384850 INFO  (closeThreadPool-12307-thread-2) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2384850 INFO  (closeThreadPool-12307-thread-2) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2384850 INFO  (closeThreadPool-12307-thread-2) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-12-19T15:45:03.751Z
   [junit4]   2> 2384854 INFO  (closeThreadPool-12307-thread-2) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2384857 INFO  (zkConnectionManagerCallback-12311-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2384857 INFO  (closeThreadPool-12307-thread-2) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2384875 INFO  (closeThreadPool-12307-thread-1) [     ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 2384875 INFO  (closeThreadPool-12307-thread-1) [     ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_B11268ACB6631A6B-001/shard-1-001/solr.xml
   [junit4]   2> 2384878 INFO  (closeThreadPool-12307-thread-1) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 2384878 INFO  (closeThreadPool-12307-thread-1) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 2384880 INFO  (closeThreadPool-12307-thread-1) [     ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@345572b9, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 2384924 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_B11268ACB6631A6B-001/shard-3-001 of type NRT
   [junit4]   2> 2384925 WARN  (closeThreadPool-12307-thread-3) [     ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
   [junit4]   2> 2384925 INFO  (closeThreadPool-12307-thread-3) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 2384925 INFO  (closeThreadPool-12307-thread-3) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 2384925 INFO  (closeThreadPool-12307-thread-3) [     ] o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z; git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 1.8.0_191-b12
   [junit4]   2> 2384926 INFO  (closeThreadPool-12307-thread-3) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 2384926 INFO  (closeThreadPool-12307-thread-3) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 2384926 INFO  (closeThreadPool-12307-thread-3) [     ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 2384927 INFO  (closeThreadPool-12307-thread-3) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@377675ca{/lf_ei/uk,null,AVAILABLE}
   [junit4]   2> 2384927 INFO  (closeThreadPool-12307-thread-3) [     ] o.e.j.s.AbstractConnector Started ServerConnector@60d0cdb7{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:34433}
   [junit4]   2> 2384927 INFO  (closeThreadPool-12307-thread-3) [     ] o.e.j.s.Server Started @2384953ms
   [junit4]   2> 2384927 INFO  (closeThreadPool-12307-thread-3) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_B11268ACB6631A6B-001/tempDir-001/jetty3, solrconfig=solrconfig.xml, hostContext=/lf_ei/uk, hostPort=34433, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_B11268ACB6631A6B-001/shard-3-001/cores}
   [junit4]   2> 2384927 ERROR (closeThreadPool-12307-thread-3) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2384927 INFO  (closeThreadPool-12307-thread-3) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 2384927 INFO  (closeThreadPool-12307-thread-3) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.5.0
   [junit4]   2> 2384927 INFO  (closeThreadPool-12307-thread-3) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2384927 INFO  (closeThreadPool-12307-thread-3) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2384927 INFO  (closeThreadPool-12307-thread-3) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-12-19T15:45:03.828Z
   [junit4]   2> 2384929 INFO  (closeThreadPool-12307-thread-3) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2384930 INFO  (zkConnectionManagerCallback-12314-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2384930 INFO  (closeThreadPool-12307-thread-3) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2384975 INFO  (closeThreadPool-12307-thread-2) [     ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 2384975 INFO  (closeThreadPool-12307-thread-2) [     ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_B11268ACB6631A6B-001/shard-2-001/solr.xml
   [junit4]   2> 2384979 INFO  (closeThreadPool-12307-thread-2) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 2384979 INFO  (closeThreadPool-12307-thread-2) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 2384980 INFO  (closeThreadPool-12307-thread-2) [     ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@345572b9, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 2385004 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 4 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_B11268ACB6631A6B-001/shard-4-001 of type NRT
   [junit4]   2> 2385005 WARN  (closeThreadPool-12307-thread-4) [     ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
   [junit4]   2> 2385005 INFO  (closeThreadPool-12307-thread-4) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 2385005 INFO  (closeThreadPool-12307-thread-4) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 2385005 INFO  (closeThreadPool-12307-thread-4) [     ] o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z; git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 1.8.0_191-b12
   [junit4]   2> 2385005 INFO  (closeThreadPool-12307-thread-4) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 2385005 INFO  (closeThreadPool-12307-thread-4) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 2385005 INFO  (closeThreadPool-12307-thread-4) [     ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 2385006 INFO  (closeThreadPool-12307-thread-4) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1383c4d8{/lf_ei/uk,null,AVAILABLE}
   [junit4]   2> 2385006 INFO  (closeThreadPool-12307-thread-4) [     ] o.e.j.s.AbstractConnector Started ServerConnector@5cdc99fe{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:44667}
   [junit4]   2> 2385006 INFO  (closeThreadPool-12307-thread-4) [     ] o.e.j.s.Server Started @2385033ms
   [junit4]   2> 2385006 INFO  (closeThreadPool-12307-thread-4) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_B11268ACB6631A6B-001/tempDir-001/jetty4, solrconfig=solrconfig.xml, hostContext=/lf_ei/uk, hostPort=44667, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_B11268ACB6631A6B-001/shard-4-001/cores}
   [junit4]   2> 2385006 ERROR (closeThreadPool-12307-thread-4) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2385006 INFO  (closeThreadPool-12307-thread-4) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 2385006 INFO  (closeThreadPool-12307-thread-4) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.5.0
   [junit4]   2> 2385006 INFO  (closeThreadPool-12307-thread-4) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2385006 INFO  (closeThreadPool-12307-thread-4) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2385007 INFO  (closeThreadPool-12307-thread-4) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-12-19T15:45:03.908Z
   [junit4]   2> 2385010 INFO  (closeThreadPool-12307-thread-4) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2385014 INFO  (zkConnectionManagerCallback-12317-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2385014 INFO  (closeThreadPool-12307-thread-4) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2385040 INFO  (closeThreadPool-12307-thread-1) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
   [junit4]   2> 2385041 WARN  (closeThreadPool-12307-thread-1) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@53a9b4ce[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2385041 WARN  (closeThreadPool-12307-thread-1) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@53a9b4ce[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2385044 WARN  (closeThreadPool-12307-thread-1) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@54fdfd30[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2385044 WARN  (closeThreadPool-12307-thread-1) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@54fdfd30[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2385045 INFO  (closeThreadPool-12307-thread-1) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39977/solr
   [junit4]   2> 2385047 INFO  (closeThreadPool-12307-thread-1) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2385049 INFO  (zkConnectionManagerCallback-12323-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2385049 INFO  (closeThreadPool-12307-thread-1) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2385076 INFO  (closeThreadPool-12307-thread-3) [     ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 2385076 INFO  (closeThreadPool-12307-thread-3) [     ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_B11268ACB6631A6B-001/shard-3-001/solr.xml
   [junit4]   2> 2385079 INFO  (closeThreadPool-12307-thread-3) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 2385079 INFO  (closeThreadPool-12307-thread-3) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 2385080 INFO  (closeThreadPool-12307-thread-3) [     ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@345572b9, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 2385089 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 5 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_B11268ACB6631A6B-001/shard-5-001 of type NRT
   [junit4]   2> 2385091 WARN  (closeThreadPool-12307-thread-5) [     ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
   [junit4]   2> 2385091 INFO  (closeThreadPool-12307-thread-5) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 2385091 INFO  (closeThreadPool-12307-thread-5) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 2385091 INFO  (closeThreadPool-12307-thread-5) [     ] o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z; git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 1.8.0_191-b12
   [junit4]   2> 2385094 INFO  (closeThreadPool-12307-thread-5) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 2385094 INFO  (closeThreadPool-12307-thread-5) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 2385094 INFO  (closeThreadPool-12307-thread-5) [     ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 2385094 INFO  (closeThreadPool-12307-thread-5) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@70ee9289{/lf_ei/uk,null,AVAILABLE}
   [junit4]   2> 2385095 INFO  (closeThreadPool-12307-thread-5) [     ] o.e.j.s.AbstractConnector Started ServerConnector@763d22a3{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:46209}
   [junit4]   2> 2385095 INFO  (closeThreadPool-12307-thread-5) [     ] o.e.j.s.Server Started @2385121ms
   [junit4]   2> 2385095 INFO  (closeThreadPool-12307-thread-5) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_B11268ACB6631A6B-001/tempDir-001/jetty5, solrconfig=solrconfig.xml, hostContext=/lf_ei/uk, hostPort=46209, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_B11268ACB6631A6B-001/shard-5-001/cores}
   [junit4]   2> 2385095 ERROR (closeThreadPool-12307-thread-5) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2385095 INFO  (closeThreadPool-12307-thread-5) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 2385095 INFO  (closeThreadPool-12307-thread-5) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.5.0
   [junit4]   2> 2385095 INFO  (closeThreadPool-12307-thread-5) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2385095 INFO  (closeThreadPool-12307-thread-5) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2385095 INFO  (closeThreadPool-12307-thread-5) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-12-19T15:45:03.996Z
   [junit4]   2> 2385097 INFO  (closeThreadPool-12307-thread-5) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2385097 INFO  (zkConnectionManagerCallback-12326-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2385097 INFO  (closeThreadPool-12307-thread-5) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2385177 INFO  (closeThreadPool-12307-thread-1) [n:127.0.0.1:39973_lf_ei%2Fuk     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2385182 INFO  (zkConnectionManagerCallback-12328-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2385182 INFO  (closeThreadPool-12307-thread-1) [n:127.0.0.1:39973_lf_ei%2Fuk     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2385185 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 6 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_B11268ACB6631A6B-001/shard-6-001 of type NRT
   [junit4]   2> 2385185 INFO  (closeThreadPool-12307-thread-1) [n:127.0.0.1:39973_lf_ei%2Fuk     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2385187 INFO  (closeThreadPool-12307-thread-3) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
   [junit4]   2> 2385188 WARN  (closeThreadPool-12307-thread-3) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@17aa6fed[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2385188 WARN  (closeThreadPool-12307-thread-3) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@17aa6fed[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2385191 WARN  (closeThreadPool-12307-thread-3) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@4a1b2965[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2385191 WARN  (closeThreadPool-12307-thread-3) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@4a1b2965[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2385191 INFO  (closeThreadPool-12307-thread-1) [n:127.0.0.1:39973_lf_ei%2Fuk     ] o.a.s.c.ZkController Publish node=127.0.0.1:39973_lf_ei%2Fuk as DOWN
   [junit4]   2> 2385192 INFO  (closeThreadPool-12307-thread-3) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39977/solr
   [junit4]   2> 2385192 INFO  (closeThreadPool-12307-thread-1) [n:127.0.0.1:39973_lf_ei%2Fuk     ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 2385192 INFO  (closeThreadPool-12307-thread-1) [n:127.0.0.1:39973_lf_ei%2Fuk     ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:39973_lf_ei%2Fuk
   [junit4]   2> 2385193 WARN  (closeThreadPool-12307-thread-6) [     ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
   [junit4]   2> 2385193 INFO  (closeThreadPool-12307-thread-6) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 2385193 INFO  (closeThreadPool-12307-thread-6) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 2385193 INFO  (closeThreadPool-12307-thread-6) [     ] o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z; git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 1.8.0_191-b12
   [junit4]   2> 2385194 INFO  (zkCallback-12290-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2385198 INFO  (closeThreadPool-12307-thread-3) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2385199 INFO  (zkCallback-12305-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2385199 INFO  (zkCallback-12327-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2385199 WARN  (closeThreadPool-12307-thread-1) [n:127.0.0.1:39973_lf_ei%2Fuk     ] o.a.s.c.CoreContainer Not all security plugins configured!  authentication=disabled authorization=disabled.  Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external.  See https://s.apache.org/solrsecurity for more info
   [junit4]   2> 2385210 INFO  (zkConnectionManagerCallback-12336-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2385210 INFO  (closeThreadPool-12307-thread-3) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2385218 INFO  (closeThreadPool-12307-thread-1) [n:127.0.0.1:39973_lf_ei%2Fuk     ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 2385220 INFO  (closeThreadPool-12307-thread-6) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 2385220 INFO  (closeThreadPool-12307-thread-6) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 2385220 INFO  (closeThreadPool-12307-thread-6) [     ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 2385222 INFO  (closeThreadPool-12307-thread-6) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@51dcbe50{/lf_ei/uk,null,AVAILABLE}
   [junit4]   2> 2385222 INFO  (closeThreadPool-12307-thread-6) [     ] o.e.j.s.AbstractConnector Started ServerConnector@331a1c78{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:43247}
   [junit4]   2> 2385222 INFO  (closeThreadPool-12307-thread-6) [     ] o.e.j.s.Server Started @2385249ms
   [junit4]   2> 2385222 INFO  (closeThreadPool-12307-thread-6) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_B11268ACB6631A6B-001/tempDir-001/jetty6, solrconfig=solrconfig.xml, hostContext=/lf_ei/uk, hostPort=43247, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_B11268ACB6631A6B-001/shard-6-001/cores}
   [junit4]   2> 2385223 ERROR (closeThreadPool-12307-thread-6) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2385223 INFO  (closeThreadPool-12307-thread-6) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 2385223 INFO  (closeThreadPool-12307-thread-6) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.5.0
   [junit4]   2> 2385223 INFO  (closeThreadPool-12307-thread-6) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2385223 INFO  (closeThreadPool-12307-thread-6) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2385223 INFO  (closeThreadPool-12307-thread-6) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-12-19T15:45:04.124Z
   [junit4]   2> 2385229 INFO  (closeThreadPool-12307-thread-6) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2385230 INFO  (zkConnectionManagerCallback-12340-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2385230 INFO  (closeThreadPool-12307-thread-6) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2385233 INFO  (closeThreadPool-12307-thread-1) [n:127.0.0.1:39973_lf_ei%2Fuk     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@345572b9
   [junit4]   2> 2385240 INFO  (closeThreadPool-12307-thread-1) [n:127.0.0.1:39973_lf_ei%2Fuk     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@345572b9
   [junit4]   2> 2385241 INFO  (closeThreadPool-12307-thread-1) [n:127.0.0.1:39973_lf_ei%2Fuk     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@345572b9
   [junit4]   2> 2385241 INFO  (closeThreadPool-12307-thread-1) [n:127.0.0.1:39973_lf_ei%2Fuk     ] o.a.s.p.PackageLoader /packages.json updated to version -1
   [junit4]   2> 2385242 INFO  (closeThreadPool-12307-thread-1) [n:127.0.0.1:39973_lf_ei%2Fuk     ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_B11268ACB6631A6B-001/shard-1-001/cores
   [junit4]   2> 2385250 INFO  (closeThreadPool-12307-thread-2) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
   [junit4]   2> 2385250 WARN  (closeThreadPool-12307-thread-2) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@7725d98b[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2385250 WARN  (closeThreadPool-12307-thread-2) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@7725d98b[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2385253 WARN  (closeThreadPool-12307-thread-2) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@2b1c8e98[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2385253 WARN  (closeThreadPool-12307-thread-2) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@2b1c8e98[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2385254 INFO  (closeThreadPool-12307-thread-2) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39977/solr
   [junit4]   2> 2385258 INFO  (closeThreadPool-12307-thread-2) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2385259 INFO  (zkConnectionManagerCallback-12346-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2385259 INFO  (closeThreadPool-12307-thread-2) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2385268 INFO  (closeThreadPool-12307-thread-1) [     ] o.a.s.c.AbstractFullDistribZkTestBase waitForLiveNode: 127.0.0.1:39973_lf_ei%2Fuk
   [junit4]   2> 2385270 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 7 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_B11268ACB6631A6B-001/shard-7-001 of type NRT
   [junit4]   2> 2385270 WARN  (closeThreadPool-12307-thread-7) [     ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
   [junit4]   2> 2385270 INFO  (closeThreadPool-12307-thread-7) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 2385270 INFO  (closeThreadPool-12307-thread-7) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 2385270 INFO  (closeThreadPool-12307-thread-7) [     ] o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z; git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 1.8.0_191-b12
   [junit4]   2> 2385272 INFO  (closeThreadPool-12307-thread-7) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 2385272 INFO  (closeThreadPool-12307-thread-7) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 2385272 INFO  (closeThreadPool-12307-thread-7) [     ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 2385272 INFO  (closeThreadPool-12307-thread-7) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@25f57a67{/lf_ei/uk,null,AVAILABLE}
   [junit4]   2> 2385272 INFO  (closeThreadPool-12307-thread-7) [     ] o.e.j.s.AbstractConnector Started ServerConnector@7c1973fc{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:41655}
   [junit4]   2> 2385272 INFO  (closeThreadPool-12307-thread-7) [     ] o.e.j.s.Server Started @2385299ms
   [junit4]   2> 2385272 INFO  (closeThreadPool-12307-thread-7) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_B11268ACB6631A6B-001/tempDir-001/jetty7, solrconfig=solrconfig.xml, hostContext=/lf_ei/uk, hostPort=41655, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_B11268ACB6631A6B-001/shard-7-001/cores}
   [junit4]   2> 2385273 ERROR (closeThreadPool-12307-thread-7) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2385273 INFO  (closeThreadPool-12307-thread-7) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 2385273 INFO  (closeThreadPool-12307-thread-7) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.5.0
   [junit4]   2> 2385273 INFO  (closeThreadPool-12307-thread-7) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2385273 INFO  (closeThreadPool-12307-thread-7) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2385273 INFO  (closeThreadPool-12307-thread-7) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-12-19T15:45:04.174Z
   [junit4]   2> 2385274 INFO  (closeThreadPool-12307-thread-7) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2385275 INFO  (zkConnectionManagerCallback-12348-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2385275 INFO  (closeThreadPool-12307-thread-7) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2385276 INFO  (closeThreadPool-12307-thread-4) [     ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 2385276 INFO  (closeThreadPool-12307-thread-4) [     ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_B11268ACB6631A6B-001/shard-4-001/solr.xml
   [junit4]   2> 2385279 INFO  (closeThreadPool-12307-thread-4) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 2385279 INFO  (closeThreadPool-12307-thread-4) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 2385281 INFO  (closeThreadPool-12307-thread-4) [     ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@345572b9, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 2385317 INFO  (closeThreadPool-12307-thread-4) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
   [junit4]   2> 2385318 WARN  (closeThreadPool-12307-thread-4) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@2fc40d5f[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2385318 WARN  (closeThreadPool-12307-thread-4) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@2fc40d5f[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2385320 WARN  (closeThreadPool-12307-thread-4) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@7ac079cf[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2385320 WARN  (closeThreadPool-12307-thread-4) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@7ac079cf[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2385321 INFO  (closeThreadPool-12307-thread-4) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39977/solr
   [junit4]   2> 2385322 INFO  (closeThreadPool-12307-thread-4) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2385322 INFO  (zkConnectionManagerCallback-12355-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2385323 INFO  (closeThreadPool-12307-thread-4) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2385350 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 8 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_B11268ACB6631A6B-001/shard-8-001 of type NRT
   [junit4]   2> 2385351 WARN  (closeThreadPool-12307-thread-1) [     ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
   [junit4]   2> 2385351 INFO  (closeThreadPool-12307-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 2385351 INFO  (closeThreadPool-12307-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 2385351 INFO  (closeThreadPool-12307-thread-1) [     ] o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z; git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 1.8.0_191-b12
   [junit4]   2> 2385351 INFO  (closeThreadPool-12307-thread-1) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 2385351 INFO  (closeThreadPool-12307-thread-1) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 2385351 INFO  (closeThreadPool-12307-thread-1) [     ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 2385352 INFO  (closeThreadPool-12307-thread-1) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1868d328{/lf_ei/uk,null,AVAILABLE}
   [junit4]   2> 2385352 INFO  (closeThreadPool-12307-thread-1) [     ] o.e.j.s.AbstractConnector Started ServerConnector@582110d3{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:41171}
   [junit4]   2> 2385352 INFO  (closeThreadPool-12307-thread-1) [     ] o.e.j.s.Server Started @2385378ms
   [junit4]   2> 2385352 INFO  (closeThreadPool-12307-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_B11268ACB6631A6B-001/tempDir-001/jetty8, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/lf_ei/uk, hostPort=41171, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_B11268ACB6631A6B-001/shard-8-001/cores}
   [junit4]   2> 2385352 ERROR (closeThreadPool-12307-thread-1) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2385352 INFO  (closeThreadPool-12307-thread-1) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 2385352 INFO  (closeThreadPool-12307-thread-1) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.5.0
   [junit4]   2> 2385352 INFO  (closeThreadPool-12307-thread-1) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2385352 INFO  (closeThreadPool-12307-thread-1) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2385352 INFO  (closeThreadPool-12307-thread-1) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-12-19T15:45:04.253Z
   [junit4]   2> 2385353 INFO  (closeThreadPool-12307-thread-1) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2385354 INFO  (zkConnectionManagerCallback-12357-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2385354 INFO  (closeThreadPool-12307-thread-1) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2385376 INFO  (closeThreadPool-12307-thread-7) [     ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 2385376 INFO  (closeThreadPool-12307-thread-7) [     ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_B11268ACB6631A6B-001/shard-7-001/solr.xml
   [junit4]   2> 2385380 INFO  (closeThreadPool-12307-thread-7) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 2385380 INFO  (closeThreadPool-12307-thread-7) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 2385381 INFO  (closeThreadPool-12307-thread-7) [     ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@345572b9, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 2385424 INFO  (closeThreadPool-12307-thread-4) [n:127.0.0.1:44667_lf_ei%2Fuk     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2385425 INFO  (zkConnectionManagerCallback-12360-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2385425 INFO  (closeThreadPool-12307-thread-4) [n:127.0.0.1:44667_lf_ei%2Fuk     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2385425 INFO  (TEST-RollingRestartTest.test-seed#[B11268ACB6631A6B]) [     ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 9 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_B11268ACB6631A6B-001/shard-9-001 of type NRT
   [junit4]   2> 2385428 INFO  (closeThreadPool-12307-thread-4) [n:127.0.0.1:44667_lf_ei%2Fuk     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2385429 INFO  (closeThreadPool-12307-thread-7) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
   [junit4]   2> 2385430 WARN  (closeThreadPool-12307-thread-7) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@32528dac[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2385430 WARN  (closeThreadPool-12307-thread-7) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@32528dac[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2385430 INFO  (closeThreadPool-12307-thread-4) [n:127.0.0.1:44667_lf_ei%2Fuk     ] o.a.s.c.ZkController Publish node=127.0.0.1:44667_lf_ei%2Fuk as DOWN
   [junit4]   2> 2385431 WARN  (closeThreadPool-12307-thread-8) [     ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size

[...truncated too long message...]

tenance
   [junit4]   2> 	2	/solr/autoscaling/events/.auto_add_replicas
   [junit4]   2> 	2	/solr/overseer/queue-work
   [junit4]   2> 
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RollingRestartTest -Dtests.method=test -Dtests.seed=B11268ACB6631A6B -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/test-data/enwiki.random.lines.txt -Dtests.locale=ga -Dtests.timezone=Pacific/Wallis -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR    145s J2 | RollingRestartTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: Timeout occurred while waiting response from server at: http://127.0.0.1:44885/lf_ei/uk
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([B11268ACB6631A6B:39465776189F7793]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:676)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:265)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:368)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:296)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.sendRequest(BaseCloudSolrClient.java:1143)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.requestWithRetryOnStaleState(BaseCloudSolrClient.java:906)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.request(BaseCloudSolrClient.java:838)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:228)
   [junit4]    > 	at org.apache.solr.cloud.RollingRestartTest.restartWithRolesTest(RollingRestartTest.java:74)
   [junit4]    > 	at org.apache.solr.cloud.RollingRestartTest.test(RollingRestartTest.java:53)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1082)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1054)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > Caused by: java.net.SocketTimeoutException: Read timed out
   [junit4]    > 	at java.net.SocketInputStream.socketRead0(Native Method)
   [junit4]    > 	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
   [junit4]    > 	at java.net.SocketInputStream.read(SocketInputStream.java:171)
   [junit4]    > 	at java.net.SocketInputStream.read(SocketInputStream.java:141)
   [junit4]    > 	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
   [junit4]    > 	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
   [junit4]    > 	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:280)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
   [junit4]    > 	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
   [junit4]    > 	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:157)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
   [junit4]    > 	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
   [junit4]    > 	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186)
   [junit4]    > 	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
   [junit4]    > 	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
   [junit4]    > 	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:564)
   [junit4]    > 	... 52 more
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_B11268ACB6631A6B-001
   [junit4]   2> Dec 19, 2019 3:47:25 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=CheapBastard, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@1d37c467), locale=ga, timezone=Pacific/Wallis
   [junit4]   2> NOTE: Linux 4.15.0-54-generic amd64/Oracle Corporation 1.8.0_191 (64-bit)/cpus=4,threads=1,free=270393048,total=534773760
   [junit4]   2> NOTE: All tests run in this JVM: [AutoAddReplicasPlanActionTest, TestSolrDeletionPolicy2, HdfsDirectoryFactoryTest, TestFreeTextSuggestions, JWTAuthPluginIntegrationTest, PeerSyncReplicationTest, TestPostingsSolrHighlighter, SolrRrdBackendFactoryTest, TestLRUCache, TestConfigSetsAPIZkFailure, TestReplicaProperties, DeleteStatusTest, SimpleCollectionCreateDeleteTest, TestDocumentBuilder, DocValuesNotIndexedTest, HdfsRecoveryZkTest, DateMathParserTest, TestXIncludeConfig, TestUtils, SolrLogAuditLoggerPluginTest, ChaosMonkeyNothingIsSafeWithPullReplicasTest, BufferStoreTest, HdfsWriteToMultipleCollectionsTest, SolrMetricManagerTest, TestJoin, TestDynamicFieldCollectionResource, TestCollationField, LegacyCloudClusterPropTest, TestNestedDocsSort, TestInPlaceUpdatesStandalone, LeaderElectionIntegrationTest, TestNonDefinedSimilarityFactory, TestMergePolicyConfig, TestFaceting, DistributedFacetPivotSmallTest, HdfsBasicDistributedZk2Test, CreateRoutedAliasTest, DistributedFacetPivotLargeTest, LukeRequestHandlerTest, SimplePostToolTest, TestJsonFacets, TestCollectionsAPIViaSolrCloudCluster, ResponseLogComponentTest, JWTVerificationkeyResolverTest, TestPayloadScoreQParserPlugin, TestStressVersions, SmileWriterTest, TestLMDirichletSimilarityFactory, TestQueryUtils, SuggesterTest, TestUtilizeNode, TestSubQueryTransformerDistrib, TestSchemaVersionResource, TestSimExtremeIndexing, SOLR749Test, TestSchemaNameResource, OverseerModifyCollectionTest, UnloadDistributedZkTest, TestLegacyTerms, TestLeaderElectionWithEmptyReplica, TestDistributedTracing, TestHttpShardHandlerFactory, TestCloudJSONFacetJoinDomain, ReturnFieldsTest, TestBulkSchemaConcurrent, CdcrOpsAndBoundariesTest, TestMacroExpander, DimensionalRoutedAliasUpdateProcessorTest, TestApiFramework, TestDistribIDF, CollectionStateFormat2Test, BigEndianAscendingWordSerializerTest, BasicAuthIntegrationTest, DistributedDebugComponentTest, TestSchemaManager, TestRequestForwarding, TestDFISimilarityFactory, TestCSVLoader, NoCacheHeaderTest, FullSolrCloudDistribCmdsTest, TestStreamBody, CoreAdminCreateDiscoverTest, SplitHandlerTest, TestDistributedGrouping, CustomHighlightComponentTest, ScheduledTriggerIntegrationTest, TestMinHashQParser, TestCloudRecovery, NodeAddedTriggerTest, SpellCheckComponentTest, AsyncCallRequestStatusResponseTest, SortByFunctionTest, TestTolerantSearch, TestSolrCoreSnapshots, SolrCloudExampleTest, BlockJoinFacetSimpleTest, CategoryRoutedAliasUpdateProcessorTest, HdfsThreadLeakTest, DistribJoinFromCollectionTest, BJQFilterAccessibleTest, TestSimpleTrackingShardHandler, UninvertDocValuesMergePolicyTest, TestAnalyzedSuggestions, TestTlogReplica, DistributedFacetSimpleRefinementLongTailTest, DocValuesMultiTest, TestQuerySenderNoQuery, TestPullReplica, WordBreakSolrSpellCheckerTest, TestSimScenario, DeleteNodeTest, ReindexCollectionTest, TaggingAttributeTest, UpdateParamsTest, TestNoOpRegenerator, TolerantUpdateProcessorTest, TestCloudConsistency, CacheHeaderTest, TestFieldCacheReopen, TestFieldSortValues, TestFieldResource, CSVRequestHandlerTest, QueryEqualityTest, TestCoreDiscovery, ImplicitSnitchTest, TestDistribDocBasedVersion, SegmentsInfoRequestHandlerTest, ExecutePlanActionTest, AlternateDirectoryTest, LeaderFailureAfterFreshStartTest, TestConfigSetProperties, TestFoldingMultitermQuery, TestSolrJacksonAnnotation, TestZkChroot, TestOmitPositions, CdcrVersionReplicationTest, TestCustomDocTransformer, TestLRUStatsCache, CollectionsAPIAsyncDistributedZkTest, HighlighterConfigTest, RandomizedTaggerTest, SpatialRPTFieldTypeTest, TestPhraseSuggestions, TestDynamicURP, RequestLoggingTest, TestSolrCloudWithDelegationTokens, BasicDistributedZkTest, CleanupOldIndexTest, CloudExitableDirectoryReaderTest, ClusterStateTest, CollectionsAPISolrJTest, DistributedQueueTest, HttpPartitionOnCommitTest, HttpPartitionTest, MoveReplicaHDFSTest, MultiSolrCloudTestCaseTest, MultiThreadedOCPTest, NodeMutatorTest, OutOfBoxZkACLAndCredentialsProvidersTest, OverriddenZkACLAndCredentialsProvidersTest, OverseerStatusTest, OverseerTaskQueueTest, RemoteQueryErrorTest, ReplaceNodeNoTargetTest, ReplaceNodeTest, RollingRestartTest]
   [junit4] Completed [474/889 (1!)] on J2 in 144.92s, 1 test, 1 error <<< FAILURES!

[...truncated 54369 lines...]

[JENKINS] Lucene-Solr-NightlyTests-8.x - Build # 299 - Still unstable

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-8.x/299/

7 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.handler.extraction.ExtractingRequestHandlerTest

Error Message:
org/w3c/dom/ElementTraversal

Stack Trace:
java.lang.NoClassDefFoundError: org/w3c/dom/ElementTraversal
	at __randomizedtesting.SeedInfo.seed([86424177E3E16DD2]:0)
	at java.lang.ClassLoader.defineClass1(Native Method)
	at java.lang.ClassLoader.defineClass(ClassLoader.java:763)
	at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:468)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:74)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:369)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:363)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:362)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:349)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at org.apache.xerces.parsers.AbstractDOMParser.startDocument(Unknown Source)
	at org.apache.xerces.xinclude.XIncludeHandler.startDocument(Unknown Source)
	at org.apache.xerces.impl.dtd.XMLDTDValidator.startDocument(Unknown Source)
	at org.apache.xerces.impl.XMLDocumentScannerImpl.startEntity(Unknown Source)
	at org.apache.xerces.impl.XMLVersionDetector.startDocumentParsing(Unknown Source)
	at org.apache.xerces.parsers.XML11Configuration.parse(Unknown Source)
	at org.apache.xerces.parsers.XML11Configuration.parse(Unknown Source)
	at org.apache.xerces.parsers.XMLParser.parse(Unknown Source)
	at org.apache.xerces.parsers.DOMParser.parse(Unknown Source)
	at org.apache.xerces.jaxp.DocumentBuilderImpl.parse(Unknown Source)
	at org.apache.solr.core.XmlConfigFile.<init>(XmlConfigFile.java:140)
	at org.apache.solr.core.XmlConfigFile.<init>(XmlConfigFile.java:87)
	at org.apache.solr.core.SolrConfig.<init>(SolrConfig.java:173)
	at org.apache.solr.core.SolrConfig.<init>(SolrConfig.java:144)
	at org.apache.solr.util.TestHarness.createConfig(TestHarness.java:89)
	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:812)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:805)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:666)
	at org.apache.solr.handler.extraction.ExtractingRequestHandlerTest.beforeClass(ExtractingRequestHandlerTest.java:55)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:878)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.ClassNotFoundException: org.w3c.dom.ElementTraversal
	at java.net.URLClassLoader.findClass(URLClassLoader.java:382)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:349)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	... 55 more


FAILED:  org.apache.solr.handler.extraction.ParseContextConfigTest.testAll

Error Message:
org/w3c/dom/ElementTraversal

Stack Trace:
java.lang.NoClassDefFoundError: org/w3c/dom/ElementTraversal
	at __randomizedtesting.SeedInfo.seed([86424177E3E16DD2:CA1C6F0E98713217]:0)
	at java.lang.ClassLoader.defineClass1(Native Method)
	at java.lang.ClassLoader.defineClass(ClassLoader.java:763)
	at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:468)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:74)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:369)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:363)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:362)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:349)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at org.apache.xerces.jaxp.DocumentBuilderImpl.newDocument(Unknown Source)
	at org.apache.solr.handler.extraction.ParseContextConfigTest.testAll(ParseContextConfigTest.java:32)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.ClassNotFoundException: org.w3c.dom.ElementTraversal
	at java.net.URLClassLoader.findClass(URLClassLoader.java:382)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:349)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	... 53 more


FAILED:  junit.framework.TestSuite.org.apache.solr.handler.extraction.TestXLSXResponseWriter

Error Message:
org/w3c/dom/ElementTraversal

Stack Trace:
java.lang.NoClassDefFoundError: org/w3c/dom/ElementTraversal
	at __randomizedtesting.SeedInfo.seed([86424177E3E16DD2]:0)
	at java.lang.ClassLoader.defineClass1(Native Method)
	at java.lang.ClassLoader.defineClass(ClassLoader.java:763)
	at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:468)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:74)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:369)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:363)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:362)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:349)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at org.apache.xerces.parsers.AbstractDOMParser.startDocument(Unknown Source)
	at org.apache.xerces.xinclude.XIncludeHandler.startDocument(Unknown Source)
	at org.apache.xerces.impl.dtd.XMLDTDValidator.startDocument(Unknown Source)
	at org.apache.xerces.impl.XMLDocumentScannerImpl.startEntity(Unknown Source)
	at org.apache.xerces.impl.XMLVersionDetector.startDocumentParsing(Unknown Source)
	at org.apache.xerces.parsers.XML11Configuration.parse(Unknown Source)
	at org.apache.xerces.parsers.XML11Configuration.parse(Unknown Source)
	at org.apache.xerces.parsers.XMLParser.parse(Unknown Source)
	at org.apache.xerces.parsers.DOMParser.parse(Unknown Source)
	at org.apache.xerces.jaxp.DocumentBuilderImpl.parse(Unknown Source)
	at org.apache.solr.core.XmlConfigFile.<init>(XmlConfigFile.java:140)
	at org.apache.solr.core.XmlConfigFile.<init>(XmlConfigFile.java:87)
	at org.apache.solr.core.SolrConfig.<init>(SolrConfig.java:173)
	at org.apache.solr.core.SolrConfig.<init>(SolrConfig.java:144)
	at org.apache.solr.util.TestHarness.createConfig(TestHarness.java:89)
	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:812)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:805)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:666)
	at org.apache.solr.handler.extraction.TestXLSXResponseWriter.beforeClass(TestXLSXResponseWriter.java:51)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:878)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.ClassNotFoundException: org.w3c.dom.ElementTraversal
	at java.net.URLClassLoader.findClass(URLClassLoader.java:382)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:349)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	... 55 more


FAILED:  junit.framework.TestSuite.org.apache.solr.handler.dataimport.TestTikaEntityProcessor

Error Message:
org/w3c/dom/ElementTraversal

Stack Trace:
java.lang.NoClassDefFoundError: org/w3c/dom/ElementTraversal
	at __randomizedtesting.SeedInfo.seed([EDCDEE1180D0EBCE]:0)
	at java.lang.ClassLoader.defineClass1(Native Method)
	at java.lang.ClassLoader.defineClass(ClassLoader.java:763)
	at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:468)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:74)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:369)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:363)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:362)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:349)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at org.apache.xerces.parsers.AbstractDOMParser.startDocument(Unknown Source)
	at org.apache.xerces.xinclude.XIncludeHandler.startDocument(Unknown Source)
	at org.apache.xerces.impl.dtd.XMLDTDValidator.startDocument(Unknown Source)
	at org.apache.xerces.impl.XMLDocumentScannerImpl.startEntity(Unknown Source)
	at org.apache.xerces.impl.XMLVersionDetector.startDocumentParsing(Unknown Source)
	at org.apache.xerces.parsers.XML11Configuration.parse(Unknown Source)
	at org.apache.xerces.parsers.XML11Configuration.parse(Unknown Source)
	at org.apache.xerces.parsers.XMLParser.parse(Unknown Source)
	at org.apache.xerces.parsers.DOMParser.parse(Unknown Source)
	at org.apache.xerces.jaxp.DocumentBuilderImpl.parse(Unknown Source)
	at org.apache.solr.core.XmlConfigFile.<init>(XmlConfigFile.java:140)
	at org.apache.solr.core.XmlConfigFile.<init>(XmlConfigFile.java:87)
	at org.apache.solr.core.SolrConfig.<init>(SolrConfig.java:173)
	at org.apache.solr.core.SolrConfig.<init>(SolrConfig.java:144)
	at org.apache.solr.util.TestHarness.createConfig(TestHarness.java:89)
	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:812)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:805)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:666)
	at org.apache.solr.handler.dataimport.TestTikaEntityProcessor.beforeClass(TestTikaEntityProcessor.java:121)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:878)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.ClassNotFoundException: org.w3c.dom.ElementTraversal
	at java.net.URLClassLoader.findClass(URLClassLoader.java:382)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:349)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	... 55 more


FAILED:  junit.framework.TestSuite.org.apache.solr.update.processor.LangDetectLanguageIdentifierUpdateProcessorFactoryTest

Error Message:
org/w3c/dom/ElementTraversal

Stack Trace:
java.lang.NoClassDefFoundError: org/w3c/dom/ElementTraversal
	at __randomizedtesting.SeedInfo.seed([8914CAB8A676D2EE]:0)
	at java.lang.ClassLoader.defineClass1(Native Method)
	at java.lang.ClassLoader.defineClass(ClassLoader.java:763)
	at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:468)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:74)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:369)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:363)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:362)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:349)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at org.apache.xerces.parsers.AbstractDOMParser.startDocument(Unknown Source)
	at org.apache.xerces.xinclude.XIncludeHandler.startDocument(Unknown Source)
	at org.apache.xerces.impl.dtd.XMLDTDValidator.startDocument(Unknown Source)
	at org.apache.xerces.impl.XMLDocumentScannerImpl.startEntity(Unknown Source)
	at org.apache.xerces.impl.XMLVersionDetector.startDocumentParsing(Unknown Source)
	at org.apache.xerces.parsers.XML11Configuration.parse(Unknown Source)
	at org.apache.xerces.parsers.XML11Configuration.parse(Unknown Source)
	at org.apache.xerces.parsers.XMLParser.parse(Unknown Source)
	at org.apache.xerces.parsers.DOMParser.parse(Unknown Source)
	at org.apache.xerces.jaxp.DocumentBuilderImpl.parse(Unknown Source)
	at org.apache.solr.core.XmlConfigFile.<init>(XmlConfigFile.java:140)
	at org.apache.solr.core.XmlConfigFile.<init>(XmlConfigFile.java:87)
	at org.apache.solr.core.SolrConfig.<init>(SolrConfig.java:173)
	at org.apache.solr.core.SolrConfig.<init>(SolrConfig.java:144)
	at org.apache.solr.util.TestHarness.createConfig(TestHarness.java:89)
	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:812)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:805)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:666)
	at org.apache.solr.update.processor.LanguageIdentifierUpdateProcessorFactoryTestCase.beforeClass(LanguageIdentifierUpdateProcessorFactoryTestCase.java:40)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:878)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.ClassNotFoundException: org.w3c.dom.ElementTraversal
	at java.net.URLClassLoader.findClass(URLClassLoader.java:382)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:349)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	... 55 more


FAILED:  junit.framework.TestSuite.org.apache.solr.update.processor.OpenNLPLangDetectUpdateProcessorFactoryTest

Error Message:
org/w3c/dom/ElementTraversal

Stack Trace:
java.lang.NoClassDefFoundError: org/w3c/dom/ElementTraversal
	at __randomizedtesting.SeedInfo.seed([8914CAB8A676D2EE]:0)
	at java.lang.ClassLoader.defineClass1(Native Method)
	at java.lang.ClassLoader.defineClass(ClassLoader.java:763)
	at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:468)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:74)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:369)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:363)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:362)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:349)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at org.apache.xerces.parsers.AbstractDOMParser.startDocument(Unknown Source)
	at org.apache.xerces.xinclude.XIncludeHandler.startDocument(Unknown Source)
	at org.apache.xerces.impl.dtd.XMLDTDValidator.startDocument(Unknown Source)
	at org.apache.xerces.impl.XMLDocumentScannerImpl.startEntity(Unknown Source)
	at org.apache.xerces.impl.XMLVersionDetector.startDocumentParsing(Unknown Source)
	at org.apache.xerces.parsers.XML11Configuration.parse(Unknown Source)
	at org.apache.xerces.parsers.XML11Configuration.parse(Unknown Source)
	at org.apache.xerces.parsers.XMLParser.parse(Unknown Source)
	at org.apache.xerces.parsers.DOMParser.parse(Unknown Source)
	at org.apache.xerces.jaxp.DocumentBuilderImpl.parse(Unknown Source)
	at org.apache.solr.core.XmlConfigFile.<init>(XmlConfigFile.java:140)
	at org.apache.solr.core.XmlConfigFile.<init>(XmlConfigFile.java:87)
	at org.apache.solr.core.SolrConfig.<init>(SolrConfig.java:173)
	at org.apache.solr.core.SolrConfig.<init>(SolrConfig.java:144)
	at org.apache.solr.util.TestHarness.createConfig(TestHarness.java:89)
	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:812)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:805)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:666)
	at org.apache.solr.update.processor.LanguageIdentifierUpdateProcessorFactoryTestCase.beforeClass(LanguageIdentifierUpdateProcessorFactoryTestCase.java:40)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:878)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.ClassNotFoundException: org.w3c.dom.ElementTraversal
	at java.net.URLClassLoader.findClass(URLClassLoader.java:382)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:349)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	... 55 more


FAILED:  junit.framework.TestSuite.org.apache.solr.update.processor.TikaLanguageIdentifierUpdateProcessorFactoryTest

Error Message:
org/w3c/dom/ElementTraversal

Stack Trace:
java.lang.NoClassDefFoundError: org/w3c/dom/ElementTraversal
	at __randomizedtesting.SeedInfo.seed([8914CAB8A676D2EE]:0)
	at java.lang.ClassLoader.defineClass1(Native Method)
	at java.lang.ClassLoader.defineClass(ClassLoader.java:763)
	at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:468)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:74)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:369)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:363)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:362)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:349)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at org.apache.xerces.parsers.AbstractDOMParser.startDocument(Unknown Source)
	at org.apache.xerces.xinclude.XIncludeHandler.startDocument(Unknown Source)
	at org.apache.xerces.impl.dtd.XMLDTDValidator.startDocument(Unknown Source)
	at org.apache.xerces.impl.XMLDocumentScannerImpl.startEntity(Unknown Source)
	at org.apache.xerces.impl.XMLVersionDetector.startDocumentParsing(Unknown Source)
	at org.apache.xerces.parsers.XML11Configuration.parse(Unknown Source)
	at org.apache.xerces.parsers.XML11Configuration.parse(Unknown Source)
	at org.apache.xerces.parsers.XMLParser.parse(Unknown Source)
	at org.apache.xerces.parsers.DOMParser.parse(Unknown Source)
	at org.apache.xerces.jaxp.DocumentBuilderImpl.parse(Unknown Source)
	at org.apache.solr.core.XmlConfigFile.<init>(XmlConfigFile.java:140)
	at org.apache.solr.core.XmlConfigFile.<init>(XmlConfigFile.java:87)
	at org.apache.solr.core.SolrConfig.<init>(SolrConfig.java:173)
	at org.apache.solr.core.SolrConfig.<init>(SolrConfig.java:144)
	at org.apache.solr.util.TestHarness.createConfig(TestHarness.java:89)
	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:812)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:805)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:666)
	at org.apache.solr.update.processor.LanguageIdentifierUpdateProcessorFactoryTestCase.beforeClass(LanguageIdentifierUpdateProcessorFactoryTestCase.java:40)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:878)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.ClassNotFoundException: org.w3c.dom.ElementTraversal
	at java.net.URLClassLoader.findClass(URLClassLoader.java:382)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:349)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	... 55 more




Build Log:
[...truncated 21972 lines...]
   [junit4] Suite: org.apache.solr.handler.dataimport.TestTikaEntityProcessor
   [junit4]   2> ERROR StatusLogger No Log4j 2 configuration file found. Using default configuration (logging only errors to the console), or user programmatically provided configurations. Set system property 'log4j2.debug' to show Log4j 2 internal initialization logging. See https://logging.apache.org/log4j/2.x/manual/configuration.html for instructions on how to configure Log4j 2
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene84): {}, docValues:{}, maxPointsInLeafNode=196, maxMBSortInHeap=5.835977338685011, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@3ada8d8), locale=en-US, timezone=America/Rankin_Inlet
   [junit4]   2> NOTE: Linux 4.15.0-54-generic amd64/Oracle Corporation 1.8.0_191 (64-bit)/cpus=4,threads=1,free=263921152,total=319291392
   [junit4]   2> NOTE: All tests run in this JVM: [TestTikaEntityProcessor]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestTikaEntityProcessor -Dtests.seed=EDCDEE1180D0EBCE -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/test-data/enwiki.random.lines.txt -Dtests.locale=en-US -Dtests.timezone=America/Rankin_Inlet -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J1 | TestTikaEntityProcessor (suite) <<<
   [junit4]    > Throwable #1: java.lang.NoClassDefFoundError: org/w3c/dom/ElementTraversal
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([EDCDEE1180D0EBCE]:0)
   [junit4]    > 	at java.lang.ClassLoader.defineClass1(Native Method)
   [junit4]    > 	at java.lang.ClassLoader.defineClass(ClassLoader.java:763)
   [junit4]    > 	at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
   [junit4]    > 	at java.net.URLClassLoader.defineClass(URLClassLoader.java:468)
   [junit4]    > 	at java.net.URLClassLoader.access$100(URLClassLoader.java:74)
   [junit4]    > 	at java.net.URLClassLoader$1.run(URLClassLoader.java:369)
   [junit4]    > 	at java.net.URLClassLoader$1.run(URLClassLoader.java:363)
   [junit4]    > 	at java.security.AccessController.doPrivileged(Native Method)
   [junit4]    > 	at java.net.URLClassLoader.findClass(URLClassLoader.java:362)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
   [junit4]    > 	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:349)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
   [junit4]    > 	at org.apache.xerces.parsers.AbstractDOMParser.startDocument(Unknown Source)
   [junit4]    > 	at org.apache.xerces.xinclude.XIncludeHandler.startDocument(Unknown Source)
   [junit4]    > 	at org.apache.xerces.impl.dtd.XMLDTDValidator.startDocument(Unknown Source)
   [junit4]    > 	at org.apache.xerces.impl.XMLDocumentScannerImpl.startEntity(Unknown Source)
   [junit4]    > 	at org.apache.xerces.impl.XMLVersionDetector.startDocumentParsing(Unknown Source)
   [junit4]    > 	at org.apache.xerces.parsers.XML11Configuration.parse(Unknown Source)
   [junit4]    > 	at org.apache.xerces.parsers.XML11Configuration.parse(Unknown Source)
   [junit4]    > 	at org.apache.xerces.parsers.XMLParser.parse(Unknown Source)
   [junit4]    > 	at org.apache.xerces.parsers.DOMParser.parse(Unknown Source)
   [junit4]    > 	at org.apache.xerces.jaxp.DocumentBuilderImpl.parse(Unknown Source)
   [junit4]    > 	at org.apache.solr.core.XmlConfigFile.<init>(XmlConfigFile.java:140)
   [junit4]    > 	at org.apache.solr.core.XmlConfigFile.<init>(XmlConfigFile.java:87)
   [junit4]    > 	at org.apache.solr.core.SolrConfig.<init>(SolrConfig.java:173)
   [junit4]    > 	at org.apache.solr.core.SolrConfig.<init>(SolrConfig.java:144)
   [junit4]    > 	at org.apache.solr.util.TestHarness.createConfig(TestHarness.java:89)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:812)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:805)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:666)
   [junit4]    > 	at org.apache.solr.handler.dataimport.TestTikaEntityProcessor.beforeClass(TestTikaEntityProcessor.java:121)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > Caused by: java.lang.ClassNotFoundException: org.w3c.dom.ElementTraversal
   [junit4]    > 	at java.net.URLClassLoader.findClass(URLClassLoader.java:382)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
   [junit4]    > 	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:349)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
   [junit4]    > 	... 55 more
   [junit4] Completed [2/2 (1!)] on J1 in 1.48s, 0 tests, 1 error <<< FAILURES!

[...truncated 1244 lines...]
   [junit4] Suite: org.apache.solr.handler.extraction.ParseContextConfigTest
   [junit4]   2> ERROR StatusLogger No Log4j 2 configuration file found. Using default configuration (logging only errors to the console), or user programmatically provided configurations. Set system property 'log4j2.debug' to show Log4j 2 internal initialization logging. See https://logging.apache.org/log4j/2.x/manual/configuration.html for instructions on how to configure Log4j 2
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ParseContextConfigTest -Dtests.method=testAll -Dtests.seed=86424177E3E16DD2 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/test-data/enwiki.random.lines.txt -Dtests.locale=ru-RU -Dtests.timezone=Australia/Perth -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   0.72s J1 | ParseContextConfigTest.testAll <<<
   [junit4]    > Throwable #1: java.lang.NoClassDefFoundError: org/w3c/dom/ElementTraversal
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([86424177E3E16DD2:CA1C6F0E98713217]:0)
   [junit4]    > 	at java.lang.ClassLoader.defineClass1(Native Method)
   [junit4]    > 	at java.lang.ClassLoader.defineClass(ClassLoader.java:763)
   [junit4]    > 	at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
   [junit4]    > 	at java.net.URLClassLoader.defineClass(URLClassLoader.java:468)
   [junit4]    > 	at java.net.URLClassLoader.access$100(URLClassLoader.java:74)
   [junit4]    > 	at java.net.URLClassLoader$1.run(URLClassLoader.java:369)
   [junit4]    > 	at java.net.URLClassLoader$1.run(URLClassLoader.java:363)
   [junit4]    > 	at java.security.AccessController.doPrivileged(Native Method)
   [junit4]    > 	at java.net.URLClassLoader.findClass(URLClassLoader.java:362)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
   [junit4]    > 	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:349)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
   [junit4]    > 	at org.apache.xerces.jaxp.DocumentBuilderImpl.newDocument(Unknown Source)
   [junit4]    > 	at org.apache.solr.handler.extraction.ParseContextConfigTest.testAll(ParseContextConfigTest.java:32)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > Caused by: java.lang.ClassNotFoundException: org.w3c.dom.ElementTraversal
   [junit4]    > 	at java.net.URLClassLoader.findClass(URLClassLoader.java:382)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
   [junit4]    > 	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:349)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
   [junit4]    > 	... 53 more
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/contrib/solr-cell/test/J1/temp/solr.handler.extraction.ParseContextConfigTest_86424177E3E16DD2-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene84): {}, docValues:{}, maxPointsInLeafNode=825, maxMBSortInHeap=7.606795981586486, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@69b31d58), locale=ru-RU, timezone=Australia/Perth
   [junit4]   2> NOTE: Linux 4.15.0-54-generic amd64/Oracle Corporation 1.8.0_191 (64-bit)/cpus=4,threads=1,free=279805304,total=319291392
   [junit4]   2> NOTE: All tests run in this JVM: [ParseContextConfigTest]
   [junit4] Completed [1/3 (1!)] on J1 in 2.93s, 1 test, 1 error <<< FAILURES!

[...truncated 1 lines...]
   [junit4] Suite: org.apache.solr.handler.extraction.ExtractingRequestHandlerTest
   [junit4]   2> ERROR StatusLogger No Log4j 2 configuration file found. Using default configuration (logging only errors to the console), or user programmatically provided configurations. Set system property 'log4j2.debug' to show Log4j 2 internal initialization logging. See https://logging.apache.org/log4j/2.x/manual/configuration.html for instructions on how to configure Log4j 2
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene84): {}, docValues:{}, maxPointsInLeafNode=791, maxMBSortInHeap=7.4232753269959195, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@2c361487), locale=ar-SY, timezone=Asia/Dubai
   [junit4]   2> NOTE: Linux 4.15.0-54-generic amd64/Oracle Corporation 1.8.0_191 (64-bit)/cpus=4,threads=1,free=213044712,total=241172480
   [junit4]   2> NOTE: All tests run in this JVM: [ExtractingRequestHandlerTest]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ExtractingRequestHandlerTest -Dtests.seed=86424177E3E16DD2 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/test-data/enwiki.random.lines.txt -Dtests.locale=ar-SY -Dtests.timezone=Asia/Dubai -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   0.00s J0 | ExtractingRequestHandlerTest (suite) <<<
   [junit4]    > Throwable #1: java.lang.NoClassDefFoundError: org/w3c/dom/ElementTraversal
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([86424177E3E16DD2]:0)
   [junit4]    > 	at java.lang.ClassLoader.defineClass1(Native Method)
   [junit4]    > 	at java.lang.ClassLoader.defineClass(ClassLoader.java:763)
   [junit4]    > 	at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
   [junit4]    > 	at java.net.URLClassLoader.defineClass(URLClassLoader.java:468)
   [junit4]    > 	at java.net.URLClassLoader.access$100(URLClassLoader.java:74)
   [junit4]    > 	at java.net.URLClassLoader$1.run(URLClassLoader.java:369)
   [junit4]    > 	at java.net.URLClassLoader$1.run(URLClassLoader.java:363)
   [junit4]    > 	at java.security.AccessController.doPrivileged(Native Method)
   [junit4]    > 	at java.net.URLClassLoader.findClass(URLClassLoader.java:362)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
   [junit4]    > 	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:349)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
   [junit4]    > 	at org.apache.xerces.parsers.AbstractDOMParser.startDocument(Unknown Source)
   [junit4]    > 	at org.apache.xerces.xinclude.XIncludeHandler.startDocument(Unknown Source)
   [junit4]    > 	at org.apache.xerces.impl.dtd.XMLDTDValidator.startDocument(Unknown Source)
   [junit4]    > 	at org.apache.xerces.impl.XMLDocumentScannerImpl.startEntity(Unknown Source)
   [junit4]    > 	at org.apache.xerces.impl.XMLVersionDetector.startDocumentParsing(Unknown Source)
   [junit4]    > 	at org.apache.xerces.parsers.XML11Configuration.parse(Unknown Source)
   [junit4]    > 	at org.apache.xerces.parsers.XML11Configuration.parse(Unknown Source)
   [junit4]    > 	at org.apache.xerces.parsers.XMLParser.parse(Unknown Source)
   [junit4]    > 	at org.apache.xerces.parsers.DOMParser.parse(Unknown Source)
   [junit4]    > 	at org.apache.xerces.jaxp.DocumentBuilderImpl.parse(Unknown Source)
   [junit4]    > 	at org.apache.solr.core.XmlConfigFile.<init>(XmlConfigFile.java:140)
   [junit4]    > 	at org.apache.solr.core.XmlConfigFile.<init>(XmlConfigFile.java:87)
   [junit4]    > 	at org.apache.solr.core.SolrConfig.<init>(SolrConfig.java:173)
   [junit4]    > 	at org.apache.solr.core.SolrConfig.<init>(SolrConfig.java:144)
   [junit4]    > 	at org.apache.solr.util.TestHarness.createConfig(TestHarness.java:89)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:812)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:805)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:666)
   [junit4]    > 	at org.apache.solr.handler.extraction.ExtractingRequestHandlerTest.beforeClass(ExtractingRequestHandlerTest.java:55)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > Caused by: java.lang.ClassNotFoundException: org.w3c.dom.ElementTraversal
   [junit4]    > 	at java.net.URLClassLoader.findClass(URLClassLoader.java:382)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
   [junit4]    > 	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:349)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
   [junit4]    > 	... 55 more
   [junit4] Completed [2/3 (2!)] on J0 in 2.97s, 0 tests, 1 error <<< FAILURES!

[...truncated 1 lines...]
   [junit4] Suite: org.apache.solr.handler.extraction.TestXLSXResponseWriter
   [junit4]   2> ERROR StatusLogger No Log4j 2 configuration file found. Using default configuration (logging only errors to the console), or user programmatically provided configurations. Set system property 'log4j2.debug' to show Log4j 2 internal initialization logging. See https://logging.apache.org/log4j/2.x/manual/configuration.html for instructions on how to configure Log4j 2
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene84): {}, docValues:{}, maxPointsInLeafNode=539, maxMBSortInHeap=6.2707063158193925, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@3e67ea6e), locale=ar-SY, timezone=US/Michigan
   [junit4]   2> NOTE: Linux 4.15.0-54-generic amd64/Oracle Corporation 1.8.0_191 (64-bit)/cpus=4,threads=1,free=284323992,total=319291392
   [junit4]   2> NOTE: All tests run in this JVM: [TestXLSXResponseWriter]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestXLSXResponseWriter -Dtests.seed=86424177E3E16DD2 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/test-data/enwiki.random.lines.txt -Dtests.locale=ar-SY -Dtests.timezone=US/Michigan -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   0.00s J2 | TestXLSXResponseWriter (suite) <<<
   [junit4]    > Throwable #1: java.lang.NoClassDefFoundError: org/w3c/dom/ElementTraversal
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([86424177E3E16DD2]:0)
   [junit4]    > 	at java.lang.ClassLoader.defineClass1(Native Method)
   [junit4]    > 	at java.lang.ClassLoader.defineClass(ClassLoader.java:763)
   [junit4]    > 	at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
   [junit4]    > 	at java.net.URLClassLoader.defineClass(URLClassLoader.java:468)
   [junit4]    > 	at java.net.URLClassLoader.access$100(URLClassLoader.java:74)
   [junit4]    > 	at java.net.URLClassLoader$1.run(URLClassLoader.java:369)
   [junit4]    > 	at java.net.URLClassLoader$1.run(URLClassLoader.java:363)
   [junit4]    > 	at java.security.AccessController.doPrivileged(Native Method)
   [junit4]    > 	at java.net.URLClassLoader.findClass(URLClassLoader.java:362)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
   [junit4]    > 	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:349)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
   [junit4]    > 	at org.apache.xerces.parsers.AbstractDOMParser.startDocument(Unknown Source)
   [junit4]    > 	at org.apache.xerces.xinclude.XIncludeHandler.startDocument(Unknown Source)
   [junit4]    > 	at org.apache.xerces.impl.dtd.XMLDTDValidator.startDocument(Unknown Source)
   [junit4]    > 	at org.apache.xerces.impl.XMLDocumentScannerImpl.startEntity(Unknown Source)
   [junit4]    > 	at org.apache.xerces.impl.XMLVersionDetector.startDocumentParsing(Unknown Source)
   [junit4]    > 	at org.apache.xerces.parsers.XML11Configuration.parse(Unknown Source)
   [junit4]    > 	at org.apache.xerces.parsers.XML11Configuration.parse(Unknown Source)
   [junit4]    > 	at org.apache.xerces.parsers.XMLParser.parse(Unknown Source)
   [junit4]    > 	at org.apache.xerces.parsers.DOMParser.parse(Unknown Source)
   [junit4]    > 	at org.apache.xerces.jaxp.DocumentBuilderImpl.parse(Unknown Source)
   [junit4]    > 	at org.apache.solr.core.XmlConfigFile.<init>(XmlConfigFile.java:140)
   [junit4]    > 	at org.apache.solr.core.XmlConfigFile.<init>(XmlConfigFile.java:87)
   [junit4]    > 	at org.apache.solr.core.SolrConfig.<init>(SolrConfig.java:173)
   [junit4]    > 	at org.apache.solr.core.SolrConfig.<init>(SolrConfig.java:144)
   [junit4]    > 	at org.apache.solr.util.TestHarness.createConfig(TestHarness.java:89)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:812)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:805)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:666)
   [junit4]    > 	at org.apache.solr.handler.extraction.TestXLSXResponseWriter.beforeClass(TestXLSXResponseWriter.java:51)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > Caused by: java.lang.ClassNotFoundException: org.w3c.dom.ElementTraversal
   [junit4]    > 	at java.net.URLClassLoader.findClass(URLClassLoader.java:382)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
   [junit4]    > 	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:349)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
   [junit4]    > 	... 55 more
   [junit4] Completed [3/3 (3!)] on J2 in 2.93s, 0 tests, 1 error <<< FAILURES!

[...truncated 1145 lines...]
   [junit4] Suite: org.apache.solr.update.processor.TikaLanguageIdentifierUpdateProcessorFactoryTest
   [junit4]   2> ERROR StatusLogger No Log4j 2 configuration file found. Using default configuration (logging only errors to the console), or user programmatically provided configurations. Set system property 'log4j2.debug' to show Log4j 2 internal initialization logging. See https://logging.apache.org/log4j/2.x/manual/configuration.html for instructions on how to configure Log4j 2
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene84): {}, docValues:{}, maxPointsInLeafNode=620, maxMBSortInHeap=6.986072571412318, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@5b981852), locale=ms, timezone=America/Cayman
   [junit4]   2> NOTE: Linux 4.15.0-54-generic amd64/Oracle Corporation 1.8.0_191 (64-bit)/cpus=4,threads=1,free=214181448,total=245366784
   [junit4]   2> NOTE: All tests run in this JVM: [TikaLanguageIdentifierUpdateProcessorFactoryTest]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TikaLanguageIdentifierUpdateProcessorFactoryTest -Dtests.seed=8914CAB8A676D2EE -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/test-data/enwiki.random.lines.txt -Dtests.locale=ms -Dtests.timezone=America/Cayman -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J0 | TikaLanguageIdentifierUpdateProcessorFactoryTest (suite) <<<
   [junit4]    > Throwable #1: java.lang.NoClassDefFoundError: org/w3c/dom/ElementTraversal
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([8914CAB8A676D2EE]:0)
   [junit4]    > 	at java.lang.ClassLoader.defineClass1(Native Method)
   [junit4]    > 	at java.lang.ClassLoader.defineClass(ClassLoader.java:763)
   [junit4]    > 	at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
   [junit4]    > 	at java.net.URLClassLoader.defineClass(URLClassLoader.java:468)
   [junit4]    > 	at java.net.URLClassLoader.access$100(URLClassLoader.java:74)
   [junit4]    > 	at java.net.URLClassLoader$1.run(URLClassLoader.java:369)
   [junit4]    > 	at java.net.URLClassLoader$1.run(URLClassLoader.java:363)
   [junit4]    > 	at java.security.AccessController.doPrivileged(Native Method)
   [junit4]    > 	at java.net.URLClassLoader.findClass(URLClassLoader.java:362)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
   [junit4]    > 	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:349)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
   [junit4]    > 	at org.apache.xerces.parsers.AbstractDOMParser.startDocument(Unknown Source)
   [junit4]    > 	at org.apache.xerces.xinclude.XIncludeHandler.startDocument(Unknown Source)
   [junit4]    > 	at org.apache.xerces.impl.dtd.XMLDTDValidator.startDocument(Unknown Source)
   [junit4]    > 	at org.apache.xerces.impl.XMLDocumentScannerImpl.startEntity(Unknown Source)
   [junit4]    > 	at org.apache.xerces.impl.XMLVersionDetector.startDocumentParsing(Unknown Source)
   [junit4]    > 	at org.apache.xerces.parsers.XML11Configuration.parse(Unknown Source)
   [junit4]    > 	at org.apache.xerces.parsers.XML11Configuration.parse(Unknown Source)
   [junit4]    > 	at org.apache.xerces.parsers.XMLParser.parse(Unknown Source)
   [junit4]    > 	at org.apache.xerces.parsers.DOMParser.parse(Unknown Source)
   [junit4]    > 	at org.apache.xerces.jaxp.DocumentBuilderImpl.parse(Unknown Source)
   [junit4]    > 	at org.apache.solr.core.XmlConfigFile.<init>(XmlConfigFile.java:140)
   [junit4]    > 	at org.apache.solr.core.XmlConfigFile.<init>(XmlConfigFile.java:87)
   [junit4]    > 	at org.apache.solr.core.SolrConfig.<init>(SolrConfig.java:173)
   [junit4]    > 	at org.apache.solr.core.SolrConfig.<init>(SolrConfig.java:144)
   [junit4]    > 	at org.apache.solr.util.TestHarness.createConfig(TestHarness.java:89)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:812)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:805)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:666)
   [junit4]    > 	at org.apache.solr.update.processor.LanguageIdentifierUpdateProcessorFactoryTestCase.beforeClass(LanguageIdentifierUpdateProcessorFactoryTestCase.java:40)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > Caused by: java.lang.ClassNotFoundException: org.w3c.dom.ElementTraversal
   [junit4]    > 	at java.net.URLClassLoader.findClass(URLClassLoader.java:382)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
   [junit4]    > 	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:349)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
   [junit4]    > 	... 55 more
   [junit4] Completed [2/4 (1!)] on J0 in 2.45s, 0 tests, 1 error <<< FAILURES!

[...truncated 1 lines...]
   [junit4] Suite: org.apache.solr.update.processor.OpenNLPLangDetectUpdateProcessorFactoryTest
   [junit4]   2> ERROR StatusLogger No Log4j 2 configuration file found. Using default configuration (logging only errors to the console), or user programmatically provided configurations. Set system property 'log4j2.debug' to show Log4j 2 internal initialization logging. See https://logging.apache.org/log4j/2.x/manual/configuration.html for instructions on how to configure Log4j 2
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene84): {}, docValues:{}, maxPointsInLeafNode=556, maxMBSortInHeap=5.405456420621814, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@392509de), locale=mt, timezone=America/Los_Angeles
   [junit4]   2> NOTE: Linux 4.15.0-54-generic amd64/Oracle Corporation 1.8.0_191 (64-bit)/cpus=4,threads=1,free=278472120,total=319291392
   [junit4]   2> NOTE: All tests run in this JVM: [OpenNLPLangDetectUpdateProcessorFactoryTest]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=OpenNLPLangDetectUpdateProcessorFactoryTest -Dtests.seed=8914CAB8A676D2EE -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/test-data/enwiki.random.lines.txt -Dtests.locale=mt -Dtests.timezone=America/Los_Angeles -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J1 | OpenNLPLangDetectUpdateProcessorFactoryTest (suite) <<<
   [junit4]    > Throwable #1: java.lang.NoClassDefFoundError: org/w3c/dom/ElementTraversal
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([8914CAB8A676D2EE]:0)
   [junit4]    > 	at java.lang.ClassLoader.defineClass1(Native Method)
   [junit4]    > 	at java.lang.ClassLoader.defineClass(ClassLoader.java:763)
   [junit4]    > 	at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
   [junit4]    > 	at java.net.URLClassLoader.defineClass(URLClassLoader.java:468)
   [junit4]    > 	at java.net.URLClassLoader.access$100(URLClassLoader.java:74)
   [junit4]    > 	at java.net.URLClassLoader$1.run(URLClassLoader.java:369)
   [junit4]    > 	at java.net.URLClassLoader$1.run(URLClassLoader.java:363)
   [junit4]    > 	at java.security.AccessController.doPrivileged(Native Method)
   [junit4]    > 	at java.net.URLClassLoader.findClass(URLClassLoader.java:362)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
   [junit4]    > 	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:349)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
   [junit4]    > 	at org.apache.xerces.parsers.AbstractDOMParser.startDocument(Unknown Source)
   [junit4]    > 	at org.apache.xerces.xinclude.XIncludeHandler.startDocument(Unknown Source)
   [junit4]    > 	at org.apache.xerces.impl.dtd.XMLDTDValidator.startDocument(Unknown Source)
   [junit4]    > 	at org.apache.xerces.impl.XMLDocumentScannerImpl.startEntity(Unknown Source)
   [junit4]    > 	at org.apache.xerces.impl.XMLVersionDetector.startDocumentParsing(Unknown Source)
   [junit4]    > 	at org.apache.xerces.parsers.XML11Configuration.parse(Unknown Source)
   [junit4]    > 	at org.apache.xerces.parsers.XML11Configuration.parse(Unknown Source)
   [junit4]    > 	at org.apache.xerces.parsers.XMLParser.parse(Unknown Source)
   [junit4]    > 	at org.apache.xerces.parsers.DOMParser.parse(Unknown Source)
   [junit4]    > 	at org.apache.xerces.jaxp.DocumentBuilderImpl.parse(Unknown Source)
   [junit4]    > 	at org.apache.solr.core.XmlConfigFile.<init>(XmlConfigFile.java:140)
   [junit4]    > 	at org.apache.solr.core.XmlConfigFile.<init>(XmlConfigFile.java:87)
   [junit4]    > 	at org.apache.solr.core.SolrConfig.<init>(SolrConfig.java:173)
   [junit4]    > 	at org.apache.solr.core.SolrConfig.<init>(SolrConfig.java:144)
   [junit4]    > 	at org.apache.solr.util.TestHarness.createConfig(TestHarness.java:89)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:812)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:805)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:666)
   [junit4]    > 	at org.apache.solr.update.processor.LanguageIdentifierUpdateProcessorFactoryTestCase.beforeClass(LanguageIdentifierUpdateProcessorFactoryTestCase.java:40)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > Caused by: java.lang.ClassNotFoundException: org.w3c.dom.ElementTraversal
   [junit4]    > 	at java.net.URLClassLoader.findClass(URLClassLoader.java:382)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
   [junit4]    > 	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:349)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
   [junit4]    > 	... 55 more
   [junit4] Completed [3/4 (2!)] on J1 in 2.30s, 0 tests, 1 error <<< FAILURES!

[...truncated 1 lines...]
   [junit4] Suite: org.apache.solr.update.processor.LangDetectLanguageIdentifierUpdateProcessorFactoryTest
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene84): {}, docValues:{}, maxPointsInLeafNode=204, maxMBSortInHeap=7.223313044814442, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@67cd18f1), locale=el-CY, timezone=America/Lower_Princes
   [junit4]   2> NOTE: Linux 4.15.0-54-generic amd64/Oracle Corporation 1.8.0_191 (64-bit)/cpus=4,threads=1,free=281875088,total=319291392
   [junit4]   2> NOTE: All tests run in this JVM: [LangDetectLanguageIdentifierUpdateProcessorFactoryTest]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LangDetectLanguageIdentifierUpdateProcessorFactoryTest -Dtests.seed=8914CAB8A676D2EE -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/test-data/enwiki.random.lines.txt -Dtests.locale=el-CY -Dtests.timezone=America/Lower_Princes -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J2 | LangDetectLanguageIdentifierUpdateProcessorFactoryTest (suite) <<<
   [junit4]    > Throwable #1: java.lang.NoClassDefFoundError: org/w3c/dom/ElementTraversal
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([8914CAB8A676D2EE]:0)
   [junit4]    > 	at java.lang.ClassLoader.defineClass1(Native Method)
   [junit4]    > 	at java.lang.ClassLoader.defineClass(ClassLoader.java:763)
   [junit4]    > 	at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
   [junit4]    > 	at java.net.URLClassLoader.defineClass(URLClassLoader.java:468)
   [junit4]    > 	at java.net.URLClassLoader.access$100(URLClassLoader.java:74)
   [junit4]    > 	at java.net.URLClassLoader$1.run(URLClassLoader.java:369)
   [junit4]    > 	at java.net.URLClassLoader$1.run(URLClassLoader.java:363)
   [junit4]    > 	at java.security.AccessController.doPrivileged(Native Method)
   [junit4]    > 	at java.net.URLClassLoader.findClass(URLClassLoader.java:362)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
   [junit4]    > 	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:349)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
   [junit4]    > 	at org.apache.xerces.parsers.AbstractDOMParser.startDocument(Unknown Source)
   [junit4]    > 	at org.apache.xerces.xinclude.XIncludeHandler.startDocument(Unknown Source)
   [junit4]    > 	at org.apache.xerces.impl.dtd.XMLDTDValidator.startDocument(Unknown Source)
   [junit4]    > 	at org.apache.xerces.impl.XMLDocumentScannerImpl.startEntity(Unknown Source)
   [junit4]    > 	at org.apache.xerces.impl.XMLVersionDetector.startDocumentParsing(Unknown Source)
   [junit4]    > 	at org.apache.xerces.parsers.XML11Configuration.parse(Unknown Source)
   [junit4]    > 	at org.apache.xerces.parsers.XML11Configuration.parse(Unknown Source)
   [junit4]    > 	at org.apache.xerces.parsers.XMLParser.parse(Unknown Source)
   [junit4]    > 	at org.apache.xerces.parsers.DOMParser.parse(Unknown Source)
   [junit4]    > 	at org.apache.xerces.jaxp.DocumentBuilderImpl.parse(Unknown Source)
   [junit4]    > 	at org.apache.solr.core.XmlConfigFile.<init>(XmlConfigFile.java:140)
   [junit4]    > 	at org.apache.solr.core.XmlConfigFile.<init>(XmlConfigFile.java:87)
   [junit4]    > 	at org.apache.solr.core.SolrConfig.<init>(SolrConfig.java:173)
   [junit4]    > 	at org.apache.solr.core.SolrConfig.<init>(SolrConfig.java:144)
   [junit4]    > 	at org.apache.solr.util.TestHarness.createConfig(TestHarness.java:89)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:812)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:805)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:666)
   [junit4]    > 	at org.apache.solr.update.processor.LanguageIdentifierUpdateProcessorFactoryTestCase.beforeClass(LanguageIdentifierUpdateProcessorFactoryTestCase.java:40)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > Caused by: java.lang.ClassNotFoundException: org.w3c.dom.ElementTraversal
   [junit4]    > 	at java.net.URLClassLoader.findClass(URLClassLoader.java:382)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
   [junit4]    > 	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:349)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
   [junit4]    > 	... 55 more
   [junit4] Completed [4/4 (3!)] on J2 in 1.17s, 0 tests, 1 error <<< FAILURES!

[...truncated 45303 lines...]

[JENKINS] Lucene-Solr-NightlyTests-8.x - Build # 298 - Failure

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-8.x/298/

3 tests failed.
FAILED:  org.apache.lucene.document.TestXYPolygonShapeQueries.testRandomBig

Error Message:
Java heap space

Stack Trace:
java.lang.OutOfMemoryError: Java heap space
	at __randomizedtesting.SeedInfo.seed([4AF56BCA12F8715A:CDA2164583A10DDA]:0)
	at org.apache.lucene.store.RAMFile.newBuffer(RAMFile.java:84)
	at org.apache.lucene.store.RAMFile.addBuffer(RAMFile.java:57)
	at org.apache.lucene.store.RAMOutputStream.switchCurrentBuffer(RAMOutputStream.java:168)
	at org.apache.lucene.store.RAMOutputStream.writeBytes(RAMOutputStream.java:154)
	at org.apache.lucene.store.MockIndexOutputWrapper.writeBytes(MockIndexOutputWrapper.java:141)
	at org.apache.lucene.util.bkd.OfflinePointWriter.append(OfflinePointWriter.java:56)
	at org.apache.lucene.util.bkd.BKDWriter.add(BKDWriter.java:247)
	at org.apache.lucene.index.RandomCodec$1$1$1.visit(RandomCodec.java:129)
	at org.apache.lucene.codecs.PointsWriter$1$1$1.visit(PointsWriter.java:117)
	at org.apache.lucene.index.AssertingLeafReader$AssertingIntersectVisitor.visit(AssertingLeafReader.java:1171)
	at org.apache.lucene.util.bkd.BKDReader.visitCompressedDocValues(BKDReader.java:735)
	at org.apache.lucene.util.bkd.BKDReader.visitDocValuesWithCardinality(BKDReader.java:684)
	at org.apache.lucene.util.bkd.BKDReader.visitDocValues(BKDReader.java:590)
	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:790)
	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:813)
	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:813)
	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:813)
	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:813)
	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:823)
	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:813)
	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:813)
	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:813)
	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:813)
	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:823)
	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:823)
	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:813)
	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:506)
	at org.apache.lucene.index.AssertingLeafReader$AssertingPointValues.intersect(AssertingLeafReader.java:1052)
	at org.apache.lucene.codecs.PointsWriter$1$1.intersect(PointsWriter.java:105)
	at org.apache.lucene.index.RandomCodec$1$1.writeField(RandomCodec.java:122)
	at org.apache.lucene.codecs.PointsWriter.mergeOneField(PointsWriter.java:62)
	at org.apache.lucene.codecs.PointsWriter.merge(PointsWriter.java:191)


FAILED:  org.apache.solr.cloud.RollingRestartTest.test

Error Message:
Address already in use

Stack Trace:
java.net.BindException: Address already in use
	at __randomizedtesting.SeedInfo.seed([CDFE17B8E4ACD0F6:45AA28624A50BD0E]:0)
	at sun.nio.ch.Net.bind0(Native Method)
	at sun.nio.ch.Net.bind(Net.java:433)
	at sun.nio.ch.Net.bind(Net.java:425)
	at sun.nio.ch.ServerSocketChannelImpl.bind(ServerSocketChannelImpl.java:223)
	at sun.nio.ch.ServerSocketAdaptor.bind(ServerSocketAdaptor.java:74)
	at org.eclipse.jetty.server.ServerConnector.openAcceptChannel(ServerConnector.java:342)
	at org.eclipse.jetty.server.ServerConnector.open(ServerConnector.java:307)
	at org.eclipse.jetty.server.AbstractNetworkConnector.doStart(AbstractNetworkConnector.java:80)
	at org.eclipse.jetty.server.ServerConnector.doStart(ServerConnector.java:231)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:72)
	at org.eclipse.jetty.server.Server.doStart(Server.java:385)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:72)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.retryOnPortBindFailure(JettySolrRunner.java:567)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:506)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:474)
	at org.apache.solr.cloud.RollingRestartTest.restartWithRolesTest(RollingRestartTest.java:104)
	at org.apache.solr.cloud.RollingRestartTest.test(RollingRestartTest.java:53)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1082)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1054)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.SyncSliceTest.test

Error Message:
expected:<5> but was:<4>

Stack Trace:
java.lang.AssertionError: expected:<5> but was:<4>
	at __randomizedtesting.SeedInfo.seed([CDFE17B8E4ACD0F6:45AA28624A50BD0E]:0)
	at org.junit.Assert.fail(Assert.java:88)
	at org.junit.Assert.failNotEquals(Assert.java:834)
	at org.junit.Assert.assertEquals(Assert.java:645)
	at org.junit.Assert.assertEquals(Assert.java:631)
	at org.apache.solr.cloud.SyncSliceTest.test(SyncSliceTest.java:150)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1082)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1054)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 11229 lines...]
   [junit4] Suite: org.apache.lucene.document.TestXYPolygonShapeQueries
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestXYPolygonShapeQueries -Dtests.method=testRandomBig -Dtests.seed=4AF56BCA12F8715A -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/test-data/enwiki.random.lines.txt -Dtests.locale=und -Dtests.timezone=America/Santa_Isabel -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR    459s J0 | TestXYPolygonShapeQueries.testRandomBig <<<
   [junit4]    > Throwable #1: java.lang.OutOfMemoryError: Java heap space
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([4AF56BCA12F8715A:CDA2164583A10DDA]:0)
   [junit4]    > 	at org.apache.lucene.store.RAMFile.newBuffer(RAMFile.java:84)
   [junit4]    > 	at org.apache.lucene.store.RAMFile.addBuffer(RAMFile.java:57)
   [junit4]    > 	at org.apache.lucene.store.RAMOutputStream.switchCurrentBuffer(RAMOutputStream.java:168)
   [junit4]    > 	at org.apache.lucene.store.RAMOutputStream.writeBytes(RAMOutputStream.java:154)
   [junit4]    > 	at org.apache.lucene.store.MockIndexOutputWrapper.writeBytes(MockIndexOutputWrapper.java:141)
   [junit4]    > 	at org.apache.lucene.util.bkd.OfflinePointWriter.append(OfflinePointWriter.java:56)
   [junit4]    > 	at org.apache.lucene.util.bkd.BKDWriter.add(BKDWriter.java:247)
   [junit4]    > 	at org.apache.lucene.index.RandomCodec$1$1$1.visit(RandomCodec.java:129)
   [junit4]    > 	at org.apache.lucene.codecs.PointsWriter$1$1$1.visit(PointsWriter.java:117)
   [junit4]    > 	at org.apache.lucene.index.AssertingLeafReader$AssertingIntersectVisitor.visit(AssertingLeafReader.java:1171)
   [junit4]    > 	at org.apache.lucene.util.bkd.BKDReader.visitCompressedDocValues(BKDReader.java:735)
   [junit4]    > 	at org.apache.lucene.util.bkd.BKDReader.visitDocValuesWithCardinality(BKDReader.java:684)
   [junit4]    > 	at org.apache.lucene.util.bkd.BKDReader.visitDocValues(BKDReader.java:590)
   [junit4]    > 	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:790)
   [junit4]    > 	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:813)
   [junit4]    > 	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:813)
   [junit4]    > 	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:813)
   [junit4]    > 	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:813)
   [junit4]    > 	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:823)
   [junit4]    > 	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:813)
   [junit4]    > 	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:813)
   [junit4]    > 	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:813)
   [junit4]    > 	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:813)
   [junit4]    > 	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:823)
   [junit4]    > 	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:823)
   [junit4]    > 	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:813)
   [junit4]    > 	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:506)
   [junit4]    > 	at org.apache.lucene.index.AssertingLeafReader$AssertingPointValues.intersect(AssertingLeafReader.java:1052)
   [junit4]    > 	at org.apache.lucene.codecs.PointsWriter$1$1.intersect(PointsWriter.java:105)
   [junit4]    > 	at org.apache.lucene.index.RandomCodec$1$1.writeField(RandomCodec.java:122)
   [junit4]    > 	at org.apache.lucene.codecs.PointsWriter.mergeOneField(PointsWriter.java:62)
   [junit4]    > 	at org.apache.lucene.codecs.PointsWriter.merge(PointsWriter.java:191)
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/lucene/build/sandbox/test/J0/temp/lucene.document.TestXYPolygonShapeQueries_4AF56BCA12F8715A-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene84): {id=PostingsFormat(name=LuceneFixedGap)}, docValues:{shape=DocValuesFormat(name=Asserting), id=DocValuesFormat(name=Lucene80)}, maxPointsInLeafNode=49, maxMBSortInHeap=7.562476951858253, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@6cbed3f8), locale=und, timezone=America/Santa_Isabel
   [junit4]   2> NOTE: Linux 4.4.0-112-generic amd64/Oracle Corporation 1.8.0_191 (64-bit)/cpus=4,threads=1,free=190916152,total=533725184
   [junit4]   2> NOTE: All tests run in this JVM: [TestIDVersionPostingsFormat, TestXYPolygonShapeQueries]
   [junit4] Completed [13/40 (1!)] on J0 in 596.98s, 5 tests, 1 error <<< FAILURES!

[...truncated 84 lines...]
   [junit4] JVM J0: stdout was not empty, see: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/lucene/build/sandbox/test/temp/junit4-J0-20191217_173133_2187461532545618962333.sysout
   [junit4] >>> JVM J0 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: Java heap space
   [junit4] Dumping heap to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/heapdumps/java_pid13812.hprof ...
   [junit4] Heap dump file created [440490531 bytes in 0.573 secs]
   [junit4] <<< JVM J0: EOF ----

[...truncated 3405 lines...]
   [junit4] Suite: org.apache.solr.cloud.SyncSliceTest
   [junit4]   2> 2360194 INFO  (SUITE-SyncSliceTest-seed#[CDFE17B8E4ACD0F6]-worker) [     ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 2360194 INFO  (SUITE-SyncSliceTest-seed#[CDFE17B8E4ACD0F6]-worker) [     ] o.a.s.SolrTestCaseJ4 Created dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SyncSliceTest_CDFE17B8E4ACD0F6-001/data-dir-82-001
   [junit4]   2> 2360194 WARN  (SUITE-SyncSliceTest-seed#[CDFE17B8E4ACD0F6]-worker) [     ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=60 numCloses=60
   [junit4]   2> 2360194 INFO  (SUITE-SyncSliceTest-seed#[CDFE17B8E4ACD0F6]-worker) [     ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 2360196 INFO  (SUITE-SyncSliceTest-seed#[CDFE17B8E4ACD0F6]-worker) [     ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 2360196 INFO  (SUITE-SyncSliceTest-seed#[CDFE17B8E4ACD0F6]-worker) [     ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
   [junit4]   2> 2360199 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 2360199 INFO  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 2360200 INFO  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 2360300 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.ZkTestServer start zk server on port:44400
   [junit4]   2> 2360300 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.ZkTestServer waitForServerUp: 127.0.0.1:44400
   [junit4]   2> 2360300 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:44400
   [junit4]   2> 2360300 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 44400
   [junit4]   2> 2360301 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2360303 INFO  (zkConnectionManagerCallback-6962-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2360303 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2360305 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2360306 INFO  (zkConnectionManagerCallback-6964-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2360306 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2360307 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 2360308 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 2360310 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 2360311 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 2360312 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 2360312 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 2360313 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 2360314 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 2360315 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 2360316 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 2360338 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 2360339 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2360340 INFO  (zkConnectionManagerCallback-6968-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2360340 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2360442 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
   [junit4]   2> 2360962 WARN  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 2360963 WARN  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
   [junit4]   2> 2360963 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 2360963 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 2360963 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z; git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 1.8.0_191-b12
   [junit4]   2> 2361012 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 2361012 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 2361012 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 2361015 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@359d004c{/,null,AVAILABLE}
   [junit4]   2> 2361016 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.e.j.s.AbstractConnector Started ServerConnector@7f845808{SSL,[ssl, http/1.1]}{127.0.0.1:35456}
   [junit4]   2> 2361024 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.e.j.s.Server Started @2361160ms
   [junit4]   2> 2361024 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SyncSliceTest_CDFE17B8E4ACD0F6-001/tempDir-001/control/data, hostContext=/, hostPort=35456, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SyncSliceTest_CDFE17B8E4ACD0F6-001/control-001/cores}
   [junit4]   2> 2361025 ERROR (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2361025 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 2361025 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.5.0
   [junit4]   2> 2361025 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2361025 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2361025 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-12-17T19:11:56.505Z
   [junit4]   2> 2361044 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2361045 INFO  (zkConnectionManagerCallback-6970-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2361045 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2361147 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 2361147 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SyncSliceTest_CDFE17B8E4ACD0F6-001/control-001/solr.xml
   [junit4]   2> 2361150 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 2361150 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 2361152 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1f7cba03, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 2362337 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
   [junit4]   2> 2362338 WARN  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
   [junit4]   2> 2362355 WARN  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@5eb65a62[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2362355 WARN  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@5eb65a62[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2362357 WARN  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
   [junit4]   2> 2362393 WARN  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@d587c10[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2362393 WARN  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@d587c10[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2362395 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:44400/solr
   [junit4]   2> 2362431 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2362432 INFO  (zkConnectionManagerCallback-6977-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2362432 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2362539 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [n:127.0.0.1:35456_     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2362558 INFO  (zkConnectionManagerCallback-6979-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2362558 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [n:127.0.0.1:35456_     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2362920 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [n:127.0.0.1:35456_     ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:35456_
   [junit4]   2> 2362921 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [n:127.0.0.1:35456_     ] o.a.s.c.Overseer Overseer (id=75920765379805189-127.0.0.1:35456_-n_0000000000) starting
   [junit4]   2> 2362935 INFO  (OverseerStateUpdate-75920765379805189-127.0.0.1:35456_-n_0000000000) [n:127.0.0.1:35456_     ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:35456_
   [junit4]   2> 2362935 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [n:127.0.0.1:35456_     ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35456_
   [junit4]   2> 2362938 WARN  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [n:127.0.0.1:35456_     ] o.a.s.c.CoreContainer Not all security plugins configured!  authentication=disabled authorization=disabled.  Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external.  See https://s.apache.org/solrsecurity for more info
   [junit4]   2> 2362953 INFO  (zkCallback-6978-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2363023 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [n:127.0.0.1:35456_     ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 2363082 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [n:127.0.0.1:35456_     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1f7cba03
   [junit4]   2> 2363128 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [n:127.0.0.1:35456_     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1f7cba03
   [junit4]   2> 2363128 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [n:127.0.0.1:35456_     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1f7cba03
   [junit4]   2> 2363129 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [n:127.0.0.1:35456_     ] o.a.s.p.PackageLoader /packages.json updated to version -1
   [junit4]   2> 2363130 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [n:127.0.0.1:35456_     ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SyncSliceTest_CDFE17B8E4ACD0F6-001/control-001/cores
   [junit4]   2> 2363200 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2363200 INFO  (zkConnectionManagerCallback-6988-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2363200 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2363210 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2363211 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:44400/solr ready
   [junit4]   2> 2363252 INFO  (qtp1726921476-18520) [n:127.0.0.1:35456_     ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:35456_&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2363272 INFO  (OverseerThreadFactory-3913-thread-1-processing-n:127.0.0.1:35456_) [n:127.0.0.1:35456_     ] o.a.s.c.a.c.CreateCollectionCmd Create collection control_collection
   [junit4]   2> 2363432 INFO  (qtp1726921476-18518) [n:127.0.0.1:35456_    x:control_collection_shard1_replica_n1 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 2363432 INFO  (qtp1726921476-18518) [n:127.0.0.1:35456_    x:control_collection_shard1_replica_n1 ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 2364618 INFO  (qtp1726921476-18518) [n:127.0.0.1:35456_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.5.0
   [junit4]   2> 2364666 INFO  (qtp1726921476-18518) [n:127.0.0.1:35456_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
   [junit4]   2> 2365132 INFO  (qtp1726921476-18518) [n:127.0.0.1:35456_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 2365181 INFO  (qtp1726921476-18518) [n:127.0.0.1:35456_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
   [junit4]   2> 2365181 INFO  (qtp1726921476-18518) [n:127.0.0.1:35456_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1f7cba03
   [junit4]   2> 2365181 INFO  (qtp1726921476-18518) [n:127.0.0.1:35456_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SyncSliceTest_CDFE17B8E4ACD0F6-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SyncSliceTest_CDFE17B8E4ACD0F6-001/control-001/cores/control_collection_shard1_replica_n1/data/]
   [junit4]   2> 2365184 INFO  (qtp1726921476-18518) [n:127.0.0.1:35456_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@36631964
   [junit4]   2> 2365186 WARN  (qtp1726921476-18518) [n:127.0.0.1:35456_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 2365375 INFO  (qtp1726921476-18518) [n:127.0.0.1:35456_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2365375 INFO  (qtp1726921476-18518) [n:127.0.0.1:35456_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2365393 INFO  (qtp1726921476-18518) [n:127.0.0.1:35456_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2365393 INFO  (qtp1726921476-18518) [n:127.0.0.1:35456_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2365394 INFO  (qtp1726921476-18518) [n:127.0.0.1:35456_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=23, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 2365394 INFO  (qtp1726921476-18518) [n:127.0.0.1:35456_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.s.SolrIndexSearcher Opening [Searcher@34dc0ce5[control_collection_shard1_replica_n1] main]
   [junit4]   2> 2365409 INFO  (qtp1726921476-18518) [n:127.0.0.1:35456_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 2365409 INFO  (qtp1726921476-18518) [n:127.0.0.1:35456_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2365410 INFO  (qtp1726921476-18518) [n:127.0.0.1:35456_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 2365410 INFO  (qtp1726921476-18518) [n:127.0.0.1:35456_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1653195324407152640
   [junit4]   2> 2365434 INFO  (searcherExecutor-3918-thread-1-processing-n:127.0.0.1:35456_ x:control_collection_shard1_replica_n1 c:control_collection s:shard1) [n:127.0.0.1:35456_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@34dc0ce5[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2365446 INFO  (qtp1726921476-18518) [n:127.0.0.1:35456_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.ZkShardTerms Successful update of terms at /collections/control_collection/terms/shard1 to Terms{values={core_node2=0}, version=0}
   [junit4]   2> 2365446 INFO  (qtp1726921476-18518) [n:127.0.0.1:35456_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/control_collection/leaders/shard1
   [junit4]   2> 2365467 INFO  (qtp1726921476-18518) [n:127.0.0.1:35456_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 2365467 INFO  (qtp1726921476-18518) [n:127.0.0.1:35456_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 2365467 INFO  (qtp1726921476-18518) [n:127.0.0.1:35456_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:35456/control_collection_shard1_replica_n1/
   [junit4]   2> 2365478 INFO  (qtp1726921476-18518) [n:127.0.0.1:35456_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 2365478 INFO  (qtp1726921476-18518) [n:127.0.0.1:35456_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.SyncStrategy https://127.0.0.1:35456/control_collection_shard1_replica_n1/ has no replicas
   [junit4]   2> 2365478 INFO  (qtp1726921476-18518) [n:127.0.0.1:35456_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/control_collection/leaders/shard1/leader after winning as /collections/control_collection/leader_elect/shard1/election/75920765379805189-core_node2-n_0000000000
   [junit4]   2> 2365480 INFO  (qtp1726921476-18518) [n:127.0.0.1:35456_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:35456/control_collection_shard1_replica_n1/ shard1
   [junit4]   2> 2365600 INFO  (zkCallback-6978-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 2365601 INFO  (qtp1726921476-18518) [n:127.0.0.1:35456_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 2365603 INFO  (qtp1726921476-18518) [n:127.0.0.1:35456_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2171
   [junit4]   2> 2365612 INFO  (qtp1726921476-18520) [n:127.0.0.1:35456_     ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
   [junit4]   2> 2365613 INFO  (OverseerCollectionConfigSetProcessor-75920765379805189-127.0.0.1:35456_-n_0000000000) [n:127.0.0.1:35456_     ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 2365742 INFO  (zkCallback-6978-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 2365742 INFO  (zkCallback-6978-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 2365770 INFO  (qtp1726921476-18520) [n:127.0.0.1:35456_     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:35456_&wt=javabin&version=2} status=0 QTime=2518
   [junit4]   2> 2365775 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.AbstractFullDistribZkTestBase Waiting to see 1 active replicas in collection: control_collection
   [junit4]   2> 2365882 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2365883 INFO  (zkConnectionManagerCallback-6994-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2365883 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2365884 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2365885 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:44400/solr ready
   [junit4]   2> 2365885 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 2365888 INFO  (qtp1726921476-18520) [n:127.0.0.1:35456_     ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=2&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2365890 INFO  (OverseerThreadFactory-3913-thread-2-processing-n:127.0.0.1:35456_) [n:127.0.0.1:35456_     ] o.a.s.c.a.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 2366095 WARN  (OverseerThreadFactory-3913-thread-2-processing-n:127.0.0.1:35456_) [n:127.0.0.1:35456_     ] o.a.s.c.a.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
   [junit4]   2> 2366097 INFO  (qtp1726921476-18520) [n:127.0.0.1:35456_     ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
   [junit4]   2> 2366098 INFO  (qtp1726921476-18520) [n:127.0.0.1:35456_     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=2&wt=javabin&version=2} status=0 QTime=210
   [junit4]   2> 2366099 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.SolrCloudTestCase active slice count: 1 expected:1
   [junit4]   2> 2366099 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.SolrCloudTestCase active replica count: 0 expected replica count: 0
   [junit4]   2> 2366108 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.SolrCloudTestCase active slice count: 1 expected:1
   [junit4]   2> 2366108 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.SolrCloudTestCase active replica count: 0 expected replica count: 0
   [junit4]   2> 2366108 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.SolrCloudTestCase active slice count: 1 expected:1
   [junit4]   2> 2366108 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.SolrCloudTestCase active replica count: 0 expected replica count: 0
   [junit4]   2> 2366108 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.AbstractFullDistribZkTestBase Creating jetty instances pullReplicaCount=0 numOtherReplicas=7
   [junit4]   2> 2366352 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SyncSliceTest_CDFE17B8E4ACD0F6-001/shard-1-001 of type NRT
   [junit4]   2> 2366354 WARN  (closeThreadPool-6995-thread-1) [     ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 2366354 WARN  (closeThreadPool-6995-thread-1) [     ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
   [junit4]   2> 2366354 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 2366354 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 2366355 INFO  (closeThreadPool-6995-thread-1) [     ] o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z; git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 1.8.0_191-b12
   [junit4]   2> 2366380 INFO  (closeThreadPool-6995-thread-1) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 2366380 INFO  (closeThreadPool-6995-thread-1) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 2366380 INFO  (closeThreadPool-6995-thread-1) [     ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 2366380 INFO  (closeThreadPool-6995-thread-1) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@32ae9993{/,null,AVAILABLE}
   [junit4]   2> 2366381 INFO  (closeThreadPool-6995-thread-1) [     ] o.e.j.s.AbstractConnector Started ServerConnector@1babf3b8{SSL,[ssl, http/1.1]}{127.0.0.1:39239}
   [junit4]   2> 2366381 INFO  (closeThreadPool-6995-thread-1) [     ] o.e.j.s.Server Started @2366517ms
   [junit4]   2> 2366381 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SyncSliceTest_CDFE17B8E4ACD0F6-001/tempDir-001/jetty1, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/, hostPort=39239, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SyncSliceTest_CDFE17B8E4ACD0F6-001/shard-1-001/cores}
   [junit4]   2> 2366381 ERROR (closeThreadPool-6995-thread-1) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2366381 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 2366382 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.5.0
   [junit4]   2> 2366382 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2366382 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2366382 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-12-17T19:12:01.862Z
   [junit4]   2> 2366383 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2366392 INFO  (zkConnectionManagerCallback-6997-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2366392 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2366493 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 2366493 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SyncSliceTest_CDFE17B8E4ACD0F6-001/shard-1-001/solr.xml
   [junit4]   2> 2366549 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 2366549 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 2366550 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1f7cba03, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 2367012 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
   [junit4]   2> 2367013 WARN  (closeThreadPool-6995-thread-1) [     ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
   [junit4]   2> 2367027 WARN  (closeThreadPool-6995-thread-1) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@19e1e314[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2367027 WARN  (closeThreadPool-6995-thread-1) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@19e1e314[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2367044 WARN  (closeThreadPool-6995-thread-1) [     ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
   [junit4]   2> 2367063 WARN  (closeThreadPool-6995-thread-1) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@e6a50d8[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2367063 WARN  (closeThreadPool-6995-thread-1) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@e6a50d8[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2367064 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:44400/solr
   [junit4]   2> 2367065 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2367081 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SyncSliceTest_CDFE17B8E4ACD0F6-001/shard-2-001 of type NRT
   [junit4]   2> 2367083 WARN  (closeThreadPool-6995-thread-2) [     ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 2367083 WARN  (closeThreadPool-6995-thread-2) [     ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
   [junit4]   2> 2367083 INFO  (closeThreadPool-6995-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 2367083 INFO  (closeThreadPool-6995-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 2367083 INFO  (closeThreadPool-6995-thread-2) [     ] o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z; git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 1.8.0_191-b12
   [junit4]   2> 2367084 INFO  (closeThreadPool-6995-thread-2) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 2367084 INFO  (closeThreadPool-6995-thread-2) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 2367084 INFO  (closeThreadPool-6995-thread-2) [     ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 2367084 INFO  (closeThreadPool-6995-thread-2) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@31d44366{/,null,AVAILABLE}
   [junit4]   2> 2367085 INFO  (closeThreadPool-6995-thread-2) [     ] o.e.j.s.AbstractConnector Started ServerConnector@eb4e44e{SSL,[ssl, http/1.1]}{127.0.0.1:44539}
   [junit4]   2> 2367085 INFO  (closeThreadPool-6995-thread-2) [     ] o.e.j.s.Server Started @2367221ms
   [junit4]   2> 2367085 INFO  (closeThreadPool-6995-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SyncSliceTest_CDFE17B8E4ACD0F6-001/tempDir-001/jetty2, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/, hostPort=44539, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SyncSliceTest_CDFE17B8E4ACD0F6-001/shard-2-001/cores}
   [junit4]   2> 2367085 ERROR (closeThreadPool-6995-thread-2) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2367085 INFO  (closeThreadPool-6995-thread-2) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 2367085 INFO  (closeThreadPool-6995-thread-2) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.5.0
   [junit4]   2> 2367085 INFO  (closeThreadPool-6995-thread-2) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2367085 INFO  (closeThreadPool-6995-thread-2) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2367085 INFO  (closeThreadPool-6995-thread-2) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-12-17T19:12:02.565Z
   [junit4]   2> 2367090 INFO  (closeThreadPool-6995-thread-2) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2367096 INFO  (zkConnectionManagerCallback-7004-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2367096 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2367132 INFO  (zkConnectionManagerCallback-7006-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2367132 INFO  (closeThreadPool-6995-thread-2) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2367235 INFO  (closeThreadPool-6995-thread-1) [n:127.0.0.1:39239_     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2367236 INFO  (zkConnectionManagerCallback-7008-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2367236 INFO  (closeThreadPool-6995-thread-1) [n:127.0.0.1:39239_     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2367249 INFO  (closeThreadPool-6995-thread-1) [n:127.0.0.1:39239_     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2367253 INFO  (closeThreadPool-6995-thread-1) [n:127.0.0.1:39239_     ] o.a.s.c.ZkController Publish node=127.0.0.1:39239_ as DOWN
   [junit4]   2> 2367254 INFO  (closeThreadPool-6995-thread-1) [n:127.0.0.1:39239_     ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 2367254 INFO  (closeThreadPool-6995-thread-1) [n:127.0.0.1:39239_     ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:39239_
   [junit4]   2> 2367267 INFO  (zkCallback-6978-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2367267 INFO  (zkCallback-7007-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2367269 WARN  (closeThreadPool-6995-thread-1) [n:127.0.0.1:39239_     ] o.a.s.c.CoreContainer Not all security plugins configured!  authentication=disabled authorization=disabled.  Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external.  See https://s.apache.org/solrsecurity for more info
   [junit4]   2> 2367291 INFO  (zkCallback-6993-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2367310 INFO  (closeThreadPool-6995-thread-1) [n:127.0.0.1:39239_     ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 2367352 INFO  (closeThreadPool-6995-thread-2) [     ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 2367352 INFO  (closeThreadPool-6995-thread-2) [     ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SyncSliceTest_CDFE17B8E4ACD0F6-001/shard-2-001/solr.xml
   [junit4]   2> 2367355 INFO  (closeThreadPool-6995-thread-2) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 2367355 INFO  (closeThreadPool-6995-thread-2) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 2367357 INFO  (closeThreadPool-6995-thread-2) [     ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1f7cba03, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 2367387 INFO  (closeThreadPool-6995-thread-1) [n:127.0.0.1:39239_     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1f7cba03
   [junit4]   2> 2367429 INFO  (closeThreadPool-6995-thread-1) [n:127.0.0.1:39239_     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1f7cba03
   [junit4]   2> 2367429 INFO  (closeThreadPool-6995-thread-1) [n:127.0.0.1:39239_     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1f7cba03
   [junit4]   2> 2367431 INFO  (closeThreadPool-6995-thread-1) [n:127.0.0.1:39239_     ] o.a.s.p.PackageLoader /packages.json updated to version -1
   [junit4]   2> 2367432 INFO  (closeThreadPool-6995-thread-1) [n:127.0.0.1:39239_     ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SyncSliceTest_CDFE17B8E4ACD0F6-001/shard-1-001/cores
   [junit4]   2> 2367516 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.c.AbstractFullDistribZkTestBase waitForLiveNode: 127.0.0.1:39239_
   [junit4]   2> 2367827 INFO  (closeThreadPool-6995-thread-2) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
   [junit4]   2> 2367828 WARN  (closeThreadPool-6995-thread-2) [     ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
   [junit4]   2> 2367829 WARN  (closeThreadPool-6995-thread-2) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@15f21f95[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2367829 WARN  (closeThreadPool-6995-thread-2) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@15f21f95[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2367831 WARN  (closeThreadPool-6995-thread-2) [     ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
   [junit4]   2> 2367840 WARN  (closeThreadPool-6995-thread-2) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@469f23f4[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2367840 WARN  (closeThreadPool-6995-thread-2) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@469f23f4[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2367841 INFO  (closeThreadPool-6995-thread-2) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:44400/solr
   [junit4]   2> 2367842 INFO  (closeThreadPool-6995-thread-2) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2367843 INFO  (zkConnectionManagerCallback-7019-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2367843 INFO  (closeThreadPool-6995-thread-2) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2367882 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SyncSliceTest_CDFE17B8E4ACD0F6-001/shard-3-001 of type NRT
   [junit4]   2> 2367883 WARN  (closeThreadPool-6995-thread-1) [     ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 2367883 WARN  (closeThreadPool-6995-thread-1) [     ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
   [junit4]   2> 2367883 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 2367883 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 2367884 INFO  (closeThreadPool-6995-thread-1) [     ] o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z; git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 1.8.0_191-b12
   [junit4]   2> 2367884 INFO  (closeThreadPool-6995-thread-1) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 2367884 INFO  (closeThreadPool-6995-thread-1) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 2367885 INFO  (closeThreadPool-6995-thread-1) [     ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 2367885 INFO  (closeThreadPool-6995-thread-1) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@33c911f2{/,null,AVAILABLE}
   [junit4]   2> 2367886 INFO  (closeThreadPool-6995-thread-1) [     ] o.e.j.s.AbstractConnector Started ServerConnector@25f6ef86{SSL,[ssl, http/1.1]}{127.0.0.1:38655}
   [junit4]   2> 2367886 INFO  (closeThreadPool-6995-thread-1) [     ] o.e.j.s.Server Started @2368021ms
   [junit4]   2> 2367886 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SyncSliceTest_CDFE17B8E4ACD0F6-001/tempDir-001/jetty3, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/, hostPort=38655, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SyncSliceTest_CDFE17B8E4ACD0F6-001/shard-3-001/cores}
   [junit4]   2> 2367886 ERROR (closeThreadPool-6995-thread-1) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2367886 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 2367886 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.5.0
   [junit4]   2> 2367886 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2367886 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2367886 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-12-17T19:12:03.366Z
   [junit4]   2> 2367891 INFO  (OverseerCollectionConfigSetProcessor-75920765379805189-127.0.0.1:35456_-n_0000000000) [n:127.0.0.1:35456_     ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 2367937 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2367937 INFO  (zkConnectionManagerCallback-7021-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2367937 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2367962 INFO  (closeThreadPool-6995-thread-2) [n:127.0.0.1:44539_     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2367963 INFO  (zkConnectionManagerCallback-7023-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2367963 INFO  (closeThreadPool-6995-thread-2) [n:127.0.0.1:44539_     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2368041 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 2368041 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SyncSliceTest_CDFE17B8E4ACD0F6-001/shard-3-001/solr.xml
   [junit4]   2> 2368045 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 2368045 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 2368046 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1f7cba03, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 2368072 INFO  (closeThreadPool-6995-thread-2) [n:127.0.0.1:44539_     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2368077 INFO  (closeThreadPool-6995-thread-2) [n:127.0.0.1:44539_     ] o.a.s.c.ZkController Publish node=127.0.0.1:44539_ as DOWN
   [junit4]   2> 2368078 INFO  (closeThreadPool-6995-thread-2) [n:127.0.0.1:44539_     ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 2368078 INFO  (closeThreadPool-6995-thread-2) [n:127.0.0.1:44539_     ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:44539_
   [junit4]   2> 2368079 INFO  (zkCallback-6993-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2368079 INFO  (zkCallback-7007-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2368079 INFO  (zkCallback-6978-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2368100 INFO  (zkCallback-7022-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2368102 WARN  (closeThreadPool-6995-thread-2) [n:127.0.0.1:44539_     ] o.a.s.c.CoreContainer Not all security plugins configured!  authentication=disabled authorization=disabled.  Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external.  See https://s.apache.org/solrsecurity for more info
   [junit4]   2> 2368198 INFO  (closeThreadPool-6995-thread-2) [n:127.0.0.1:44539_     ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 2368355 INFO  (closeThreadPool-6995-thread-2) [n:127.0.0.1:44539_     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1f7cba03
   [junit4]   2> 2368426 INFO  (closeThreadPool-6995-thread-2) [n:127.0.0.1:44539_     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1f7cba03
   [junit4]   2> 2368426 INFO  (closeThreadPool-6995-thread-2) [n:127.0.0.1:44539_     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1f7cba03
   [junit4]   2> 2368433 INFO  (closeThreadPool-6995-thread-2) [n:127.0.0.1:44539_     ] o.a.s.p.PackageLoader /packages.json updated to version -1
   [junit4]   2> 2368451 INFO  (closeThreadPool-6995-thread-2) [n:127.0.0.1:44539_     ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SyncSliceTest_CDFE17B8E4ACD0F6-001/shard-2-001/cores
   [junit4]   2> 2368670 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
   [junit4]   2> 2368671 WARN  (closeThreadPool-6995-thread-1) [     ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
   [junit4]   2> 2368771 WARN  (closeThreadPool-6995-thread-1) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@18479d1b[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2368771 WARN  (closeThreadPool-6995-thread-1) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@18479d1b[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2368774 INFO  (closeThreadPool-6995-thread-2) [     ] o.a.s.c.AbstractFullDistribZkTestBase waitForLiveNode: 127.0.0.1:44539_
   [junit4]   2> 2368885 WARN  (closeThreadPool-6995-thread-1) [     ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
   [junit4]   2> 2368961 WARN  (closeThreadPool-6995-thread-1) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@39dc0329[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2368961 WARN  (closeThreadPool-6995-thread-1) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@39dc0329[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2368975 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:44400/solr
   [junit4]   2> 2369028 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2369033 INFO  (zkConnectionManagerCallback-7034-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2369033 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2369170 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2369190 INFO  (zkConnectionManagerCallback-7036-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2369190 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2369191 INFO  (TEST-SyncSliceTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 4 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SyncSliceTest_CDFE17B8E4ACD0F6-001/shard-4-001 of type NRT
   [junit4]   2> 2369195 WARN  (closeThreadPool-6995-thread-2) [     ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 2369196 WARN  (closeThreadPool-6995-thread-2) [     ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
   [junit4]   2> 2369196 INFO  (closeThreadPool-6995-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 2369196 INFO  (closeThreadPool-6995-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 2369196 INFO  (closeThreadPool-6995-thread-2) [     ] o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z; git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 1.8.0_191-b12
   [junit4]   2> 2369234 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 2369295 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.c.ZkController Publish node=127.0.0.1:38655_ as DOWN
   [junit4]   2> 2369313 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 2369314 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38655_
   [junit4]   2> 2369333 INFO  (closeThreadPool-6995-thread-2) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 2369333 INFO  (closeThreadPool-6995-thread-2) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 2369333 INFO  (closeThreadPool-6995-thread-2) [     ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 2369334 INFO  (closeThreadPool-6995-thread-2) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@16a0a56b{/,null,AVAILABLE}
   [junit4]   2> 2369334 INFO  (closeThreadPool-6995-thread-2) [     ] o.e.j.s.AbstractConnector Started ServerConnector@10053159{SSL,[ssl, http/1.1]}{127.0.0.1:41791}
   [junit4]   2> 2369334 INFO  (closeThreadPool-6995-thread-2) [     ] o.e.j.s.Server Started @2369470ms
   [junit4]   2> 2369334 INFO  (closeThreadPool-6995-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SyncSliceTest_CDFE17B8E4ACD0F6-001/tempDir-001/jetty4, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/, hostPort=41791, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SyncSliceTest_CDFE17B8E4ACD0F6-001/shard-4-001/cores}
   [junit4]   2> 2369335 ERROR (closeThreadPool-6995-thread-2) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2369335 INFO  (closeThreadPool-6995-thread-2) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 2369335 INFO  (closeThreadPool-6995-thread-2) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.5.0
   [junit4]   2> 2369335 INFO  (closeThreadPool-6995-thread-2) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2369335 INFO  (closeThreadPool-6995-thread-2) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2369335 INFO  (closeThreadPool-6995-thread-2) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-12-17T19:12:04.815Z
   [junit4]   2> 2369361 INFO  (zkCallback-6993-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 2369361 INFO  (zkCallback-7035-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 2369361 INFO  (closeThreadPool-6995-thread-2) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2369368 INFO  (zkCallback-7022-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 2369368 INFO  (zkCallback-7007-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 2369397 INFO  (zkCallback-6978-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 2369417 INFO  (zkConnectionManagerCallback-7040-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2369418 INFO  (closeThreadPool-6995-thread-2) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2369483 WARN  (closeThreadPool-6995-thread-1) [     ] o.a.s.c.CoreContainer Not all security plugins configured!  authentication=disabled authorization=disabled.  Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external.  See https://s.apache.org/solrsecurity for more info
   [junit4]   2> 2369623 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 2369645 INFO  (closeThreadPool-6995-thread-2) [     ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 2369645 INFO  (closeThreadPool-6995-thread-2) [     ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SyncSliceTest_CDFE17B8E4ACD0F6-001/shard-4-001/solr.xml
   [junit4]   2> 2369648 INFO  (closeThreadPool-6995-thread-2) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 2369648 INFO  (closeThreadPool-6995-thread-2) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 2369682 INFO  (closeThreadPool-6995-thread-2) [     ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1f7cba03, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 2369734 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1f7cba03
   [junit4]   2> 2369787 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1f7cba03
   [junit4]   2> 2369788 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1f7cba03
   [junit4]   2> 2369836 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.p.PackageLoader /packages.json updated to version -1
   [junit4]   2> 2369837 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SyncSliceTest_CDFE17B8E4ACD0F6-001/shard-3-001/cores
   [junit4]   2> 2370069 INFO  (closeThreadPool-6995-thread-2) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
   [junit4]   2> 2370069 WARN  (closeThreadPool-6995-thread-2) [     ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
   [junit4]   2> 2370102 INFO  (closeThreadPool-6995-thread-1) [     ] o.a.s.c.AbstractFullDistribZkTestBase waitForLiveNode: 127.0.0.1:38655_
   [junit4]   2> 2370243 WARN  (closeThreadPool-6995-thread-2) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@552fa0f1[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2370243 WARN  (closeThreadPool-6995-thread-2) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@552fa0f1[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2370280 WARN  (closeThreadPool-6995-thread-2) [     ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
   [junit4]   2> 2370281 WARN  (closeThreadPool-6995-thread-2) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@5800f264[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2370281 WARN  (closeThreadPool-6995-thread-2) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@5800f264[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2370282 INFO  (closeThreadPool-6995-thread-2) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:44400/solr
   [junit4]   2> 2370300 INFO  (closeThreadPool-6995-thread-2) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2370377 INFO  (zkConnectionManagerCallback-7049-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2370377 INFO  (closeThreadPool-6995-thread-2)

[...truncated too long message...]

ction/75920905749069869-127.0.0.1:38477_-n_0000000033
   [junit4]   2> 	2	/solr/collections/collection1/leader_elect/shard2/election/75920905749069877-core_node32-n_0000000007
   [junit4]   2> 	2	/solr/overseer_elect/election/75920905749069897-127.0.0.1:42079_-n_0000000029
   [junit4]   2> 	2	/solr/collections/collection1/leader_elect/shard1/election/75920905749069870-core_node19-n_0000000004
   [junit4]   2> 	2	/solr/collections/collection1/leader_elect/shard1/election/75920905749069912-core_node23-n_0000000013
   [junit4]   2> 	2	/solr/collections/collection1/leader_elect/shard1/election/75920905749069885-core_node27-n_0000000008
   [junit4]   2> 	2	/solr/overseer_elect/election/75920905749069829-127.0.0.1:46383_-n_0000000000
   [junit4]   2> 	2	/solr/overseer_elect/election/75920905749069854-127.0.0.1:35793_-n_0000000051
   [junit4]   2> 	2	/solr/overseer_elect/election/75920905749069866-127.0.0.1:42382_-n_0000000010
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	1234	/solr/live_nodes
   [junit4]   2> 	406	/solr/overseer/queue
   [junit4]   2> 	152	/solr/overseer/collection-queue-work
   [junit4]   2> 	45	/solr/collections
   [junit4]   2> 	25	/solr/autoscaling/events/.auto_add_replicas
   [junit4]   2> 	24	/solr/autoscaling/events/.scheduled_maintenance
   [junit4]   2> 	24	/solr/overseer/queue-work
   [junit4]   2> 
   [junit4]   2> 4777824 INFO  (TEST-RollingRestartTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.ZkTestServer waitForServerDown: 127.0.0.1:34383
   [junit4]   2> 4777825 INFO  (TEST-RollingRestartTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:34383
   [junit4]   2> 4777825 INFO  (TEST-RollingRestartTest.test-seed#[CDFE17B8E4ACD0F6]) [     ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 34383
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RollingRestartTest -Dtests.method=test -Dtests.seed=CDFE17B8E4ACD0F6 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/test-data/enwiki.random.lines.txt -Dtests.locale=es-US -Dtests.timezone=UCT -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR    276s J1 | RollingRestartTest.test <<<
   [junit4]    > Throwable #1: java.net.BindException: Address already in use
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([CDFE17B8E4ACD0F6:45AA28624A50BD0E]:0)
   [junit4]    > 	at sun.nio.ch.Net.bind0(Native Method)
   [junit4]    > 	at sun.nio.ch.Net.bind(Net.java:433)
   [junit4]    > 	at sun.nio.ch.Net.bind(Net.java:425)
   [junit4]    > 	at sun.nio.ch.ServerSocketChannelImpl.bind(ServerSocketChannelImpl.java:223)
   [junit4]    > 	at sun.nio.ch.ServerSocketAdaptor.bind(ServerSocketAdaptor.java:74)
   [junit4]    > 	at org.eclipse.jetty.server.ServerConnector.openAcceptChannel(ServerConnector.java:342)
   [junit4]    > 	at org.eclipse.jetty.server.ServerConnector.open(ServerConnector.java:307)
   [junit4]    > 	at org.eclipse.jetty.server.AbstractNetworkConnector.doStart(AbstractNetworkConnector.java:80)
   [junit4]    > 	at org.eclipse.jetty.server.ServerConnector.doStart(ServerConnector.java:231)
   [junit4]    > 	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:72)
   [junit4]    > 	at org.eclipse.jetty.server.Server.doStart(Server.java:385)
   [junit4]    > 	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:72)
   [junit4]    > 	at org.apache.solr.client.solrj.embedded.JettySolrRunner.retryOnPortBindFailure(JettySolrRunner.java:567)
   [junit4]    > 	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:506)
   [junit4]    > 	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:474)
   [junit4]    > 	at org.apache.solr.cloud.RollingRestartTest.restartWithRolesTest(RollingRestartTest.java:104)
   [junit4]    > 	at org.apache.solr.cloud.RollingRestartTest.test(RollingRestartTest.java:53)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1082)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1054)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.RollingRestartTest_CDFE17B8E4ACD0F6-001
   [junit4]   2> Dec 17, 2019 7:52:13 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene84): {}, docValues:{}, maxPointsInLeafNode=1333, maxMBSortInHeap=6.003305498245087, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@38994636), locale=es-US, timezone=UCT
   [junit4]   2> NOTE: Linux 4.4.0-112-generic amd64/Oracle Corporation 1.8.0_191 (64-bit)/cpus=4,threads=1,free=162307624,total=527958016
   [junit4]   2> NOTE: All tests run in this JVM: [ShardsWhitelistTest, TestSearcherReuse, TestSimPolicyCloud, TestInfoStreamLogging, TestRealTimeGet, TestClassicSimilarityFactory, DebugComponentTest, MinimalSchemaTest, BasicDistributedZk2Test, TestCopyFieldCollectionResource, TestManagedSynonymGraphFilterFactory, DOMUtilTest, TestLocalFSCloudBackupRestore, TestBlobHandler, RecoveryZkTest, BadComponentTest, SolrInfoBeanTest, TestRestoreCore, CurrencyFieldTypeTest, ClassificationUpdateProcessorIntegrationTest, TestCloudSearcherWarming, ChaosMonkeyNothingIsSafeTest, TestSystemCollAutoCreate, UUIDUpdateProcessorFallbackTest, TestPrepRecovery, SSLMigrationTest, DistributedVersionInfoTest, CreateCollectionCleanupTest, TestWithCollection, TestRebalanceLeaders, HLLUtilTest, TestSolrConfigHandlerCloud, TestDelegationWithHadoopAuth, CloudExitableDirectoryReaderTest, TestUniqueKeyFieldResource, TestSegmentSorting, TestQueryTypes, TestSolrCoreParser, HdfsLockFactoryTest, TestRandomFaceting, TestCoreBackup, TestSolrXml, CdcrRequestHandlerTest, SuggesterTSTTest, SolrCoreTest, CdcrBootstrapTest, SolrCoreMetricManagerTest, PeerSyncWithBufferUpdatesTest, TestFileDictionaryLookup, IndexSizeTriggerTest, SuggestComponentContextFilterQueryTest, TestQueryingOnDownCollection, TestDynamicLoadingUrl, DocExpirationUpdateProcessorFactoryTest, OverseerStatusTest, ExplicitHLLTest, DisMaxRequestHandlerTest, AuditLoggerPluginTest, TestSlowCompositeReaderWrapper, CollectionPropsTest, TestRandomCollapseQParserPlugin, SplitShardTest, HdfsChaosMonkeyNothingIsSafeTest, IgnoreLargeDocumentProcessorFactoryTest, SpellCheckCollatorWithCollapseTest, TestChildDocTransformerHierarchy, ComputePlanActionTest, ConcurrentCreateRoutedAliasTest, SolrMetricsIntegrationTest, TestDefaultStatsCache, DistribDocExpirationUpdateProcessorTest, TestLegacyNumericUtils, TestAnalyzeInfixSuggestions, BlockJoinFacetRandomTest, TestObjectReleaseTracker, CollectionTooManyReplicasTest, CurrencyRangeFacetCloudTest, SystemInfoHandlerTest, NumericFieldsTest, TestCodecSupport, TermVectorComponentTest, TestCloudInspectUtil, TestShardHandlerFactory, PeerSyncWithLeaderTest, TestPKIAuthenticationPlugin, LeaderVoteWaitTimeoutTest, TestMultiWordSynonyms, ZkNodePropsTest, TestCustomStream, TestJsonFacetsWithNestedObjects, TestSchemaField, SolrIndexSplitterTest, AssignTest, NodeMutatorTest, DistributedQueryElevationComponentTest, CoreAdminOperationTest, TestWordDelimiterFilterFactory, RankQueryTest, BasicAuthStandaloneTest, TestDeprecatedFilters, MultiSolrCloudTestCaseTest, NodeAddedTriggerIntegrationTest, TestSolrCloudWithSecureImpersonation, AtomicUpdatesTest, TestCloudSchemaless, DistanceFunctionTest, ExitableDirectoryReaderTest, TestLockTree, XmlUpdateRequestHandlerTest, MetricsHistoryWithAuthIntegrationTest, TestFacetMethods, TestQuerySenderListener, BinaryUpdateRequestHandlerTest, HdfsUnloadDistributedZkTest, ForceLeaderWithTlogReplicasTest, DistributedFacetExistsSmallTest, TestHalfAndHalfDocValues, TestSolrQueryParser, PathHierarchyTokenizerFactoryTest, NodeLostTriggerTest, MoreLikeThisComponentTest, TestClusterStateMutator, SecurityConfHandlerTest, IndexSchemaTest, TestNumericRangeQuery32, UtilsToolTest, HighlighterTest, MetricUtilsTest, AddSchemaFieldsUpdateProcessorFactoryTest, TestRetrieveFieldsOptimizer, ResponseBuilderTest, SolrRequestParserTest, OverseerCollectionConfigSetProcessorTest, HdfsSyncSliceTest, EmbeddedSolrNoSerializeTest, SortSpecParsingTest, ConcurrentDeleteAndCreateCollectionTest, TestCoreContainer, SolrJmxReporterTest, ChaosMonkeySafeLeaderWithPullReplicasTest, HdfsRecoverLeaseTest, TestWriterPerf, TestComplexPhraseQParserPlugin, BasicFunctionalityTest, ConvertedLegacyTest, SampleTest, TestCursorMarkWithoutUniqueKey, TestDistributedGrouping, TestDistributedSearch, DeleteShardTest, DeleteStatusTest, ForceLeaderTest, HttpPartitionWithTlogReplicasTest, RollingRestartTest]
   [junit4] Completed [524/889 (2!)] on J1 in 276.56s, 1 test, 1 error <<< FAILURES!

[...truncated 10779 lines...]
BUILD FAILED
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/build.xml:652: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/build.xml:586: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid13812.hprof

Total time: 504 minutes 28 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any

[JENKINS] Lucene-Solr-NightlyTests-8.x - Build # 297 - Still unstable

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-8.x/297/

1 tests failed.
FAILED:  org.apache.lucene.index.TestDemoParallelLeafReader.testRandomMultipleSchemaGens

Error Message:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/lucene/build/core/test/J2/temp/lucene.index.TestDemoParallelLeafReader_D1AD2D71F762344B-001/tempDir-002/index/_ms.fdx: Too many open files

Stack Trace:
java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/lucene/build/core/test/J2/temp/lucene.index.TestDemoParallelLeafReader_D1AD2D71F762344B-001/tempDir-002/index/_ms.fdx: Too many open files
	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
	at org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:197)
	at org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:202)
	at java.nio.channels.FileChannel.open(FileChannel.java:287)
	at java.nio.channels.FileChannel.open(FileChannel.java:335)
	at org.apache.lucene.store.NIOFSDirectory.openInput(NIOFSDirectory.java:81)
	at org.apache.lucene.store.NRTCachingDirectory.openInput(NRTCachingDirectory.java:181)
	at org.apache.lucene.util.LuceneTestCase.slowFileExists(LuceneTestCase.java:2862)
	at org.apache.lucene.store.MockDirectoryWrapper.openInput(MockDirectoryWrapper.java:747)
	at org.apache.lucene.store.Directory.openChecksumInput(Directory.java:157)
	at org.apache.lucene.store.MockDirectoryWrapper.openChecksumInput(MockDirectoryWrapper.java:1069)
	at org.apache.lucene.codecs.compressing.CompressingStoredFieldsReader.<init>(CompressingStoredFieldsReader.java:128)
	at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsReader(CompressingStoredFieldsFormat.java:121)
	at org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsReader(Lucene50StoredFieldsFormat.java:173)
	at org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsReader(AssertingStoredFieldsFormat.java:43)
	at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:127)
	at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:84)
	at org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:177)
	at org.apache.lucene.index.BufferedUpdatesStream$SegmentState.<init>(BufferedUpdatesStream.java:271)
	at org.apache.lucene.index.FrozenBufferedUpdates.openSegmentStates(FrozenBufferedUpdates.java:328)
	at org.apache.lucene.index.FrozenBufferedUpdates.forceApply(FrozenBufferedUpdates.java:221)
	at org.apache.lucene.index.FrozenBufferedUpdates.tryApply(FrozenBufferedUpdates.java:159)
	at org.apache.lucene.index.IndexWriter.lambda$publishFrozenUpdates$3(IndexWriter.java:2592)
	at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5116)
	at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1597)
	at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1213)
	at org.apache.lucene.index.TestDemoParallelLeafReader.testRandomMultipleSchemaGens(TestDemoParallelLeafReader.java:980)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 1556 lines...]
   [junit4] Suite: org.apache.lucene.index.TestDemoParallelLeafReader
   [junit4]   2> ??? 16, 2019 1:22:39 ?? com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
   [junit4]   2> ??: Uncaught exception in thread: Thread[Lucene Merge Thread #56,5,TGRP-TestDemoParallelLeafReader]
   [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.lang.IllegalStateException: this writer hit an unrecoverable error; cannot complete merge
   [junit4]   2> 	at __randomizedtesting.SeedInfo.seed([D1AD2D71F762344B]:0)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
   [junit4]   2> Caused by: java.lang.IllegalStateException: this writer hit an unrecoverable error; cannot complete merge
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.commitMerge(IndexWriter.java:3880)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4619)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
   [junit4]   2> Caused by: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/lucene/build/core/test/J2/temp/lucene.index.TestDemoParallelLeafReader_D1AD2D71F762344B-001/tempDir-002/segs/4co6e0ws25r8xxi24lsus808g_121/_g1.fdx: Too many open files
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.newByteChannel(HandleTrackingFS.java:271)
   [junit4]   2> 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newByteChannel(FilterFileSystemProvider.java:212)
   [junit4]   2> 	at java.nio.file.Files.newByteChannel(Files.java:361)
   [junit4]   2> 	at java.nio.file.Files.newByteChannel(Files.java:407)
   [junit4]   2> 	at org.apache.lucene.store.SimpleFSDirectory.openInput(SimpleFSDirectory.java:77)
   [junit4]   2> 	at org.apache.lucene.util.LuceneTestCase.slowFileExists(LuceneTestCase.java:2862)
   [junit4]   2> 	at org.apache.lucene.store.MockDirectoryWrapper.openInput(MockDirectoryWrapper.java:747)
   [junit4]   2> 	at org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100)
   [junit4]   2> 	at org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100)
   [junit4]   2> 	at org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100)
   [junit4]   2> 	at org.apache.lucene.store.Directory.openChecksumInput(Directory.java:157)
   [junit4]   2> 	at org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.write(Lucene50CompoundFormat.java:89)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.createCompoundFile(IndexWriter.java:5014)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriterPerThread.sealFlushedSegment(DocumentsWriterPerThread.java:574)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:513)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:555)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriter.postUpdate(DocumentsWriter.java:414)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:515)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1594)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1213)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$2.reindex(TestDemoParallelLeafReader.java:730)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:405)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:163)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.wrapForMerge(TestDemoParallelLeafReader.java:533)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4417)
   [junit4]   2> 	... 3 more
   [junit4]   2> 
   [junit4]   2> ??? 16, 2019 1:22:39 ?? com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
   [junit4]   2> ??: Uncaught exception in thread: Thread[Lucene Merge Thread #54,5,TGRP-TestDemoParallelLeafReader]
   [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/lucene/build/core/test/J2/temp/lucene.index.TestDemoParallelLeafReader_D1AD2D71F762344B-001/tempDir-002/segs/4co6e0ws25r8xxi24lsus808g_121/_g1.fdx: Too many open files
   [junit4]   2> 	at __randomizedtesting.SeedInfo.seed([D1AD2D71F762344B]:0)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
   [junit4]   2> Caused by: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/lucene/build/core/test/J2/temp/lucene.index.TestDemoParallelLeafReader_D1AD2D71F762344B-001/tempDir-002/segs/4co6e0ws25r8xxi24lsus808g_121/_g1.fdx: Too many open files
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.newByteChannel(HandleTrackingFS.java:271)
   [junit4]   2> 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newByteChannel(FilterFileSystemProvider.java:212)
   [junit4]   2> 	at java.nio.file.Files.newByteChannel(Files.java:361)
   [junit4]   2> 	at java.nio.file.Files.newByteChannel(Files.java:407)
   [junit4]   2> 	at org.apache.lucene.store.SimpleFSDirectory.openInput(SimpleFSDirectory.java:77)
   [junit4]   2> 	at org.apache.lucene.util.LuceneTestCase.slowFileExists(LuceneTestCase.java:2862)
   [junit4]   2> 	at org.apache.lucene.store.MockDirectoryWrapper.openInput(MockDirectoryWrapper.java:747)
   [junit4]   2> 	at org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100)
   [junit4]   2> 	at org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100)
   [junit4]   2> 	at org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100)
   [junit4]   2> 	at org.apache.lucene.store.Directory.openChecksumInput(Directory.java:157)
   [junit4]   2> 	at org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.write(Lucene50CompoundFormat.java:89)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.createCompoundFile(IndexWriter.java:5014)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriterPerThread.sealFlushedSegment(DocumentsWriterPerThread.java:574)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:513)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:555)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriter.postUpdate(DocumentsWriter.java:414)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:515)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1594)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1213)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$2.reindex(TestDemoParallelLeafReader.java:730)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:405)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:163)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.wrapForMerge(TestDemoParallelLeafReader.java:533)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4417)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
   [junit4]   2> 
   [junit4]   2> ??? 16, 2019 1:22:39 ?? com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
   [junit4]   2> ??: Uncaught exception in thread: Thread[Lucene Merge Thread #72,5,TGRP-TestDemoParallelLeafReader]
   [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/lucene/build/core/test/J2/temp/lucene.index.TestDemoParallelLeafReader_D1AD2D71F762344B-001/tempDir-002/segs/4co6e0ws25r8xxi24lsus85d6_122/_1_Lucene80_0.dvm: Too many open files
   [junit4]   2> 	at __randomizedtesting.SeedInfo.seed([D1AD2D71F762344B]:0)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
   [junit4]   2> Caused by: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/lucene/build/core/test/J2/temp/lucene.index.TestDemoParallelLeafReader_D1AD2D71F762344B-001/tempDir-002/segs/4co6e0ws25r8xxi24lsus85d6_122/_1_Lucene80_0.dvm: Too many open files
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
   [junit4]   2> 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
   [junit4]   2> 	at java.nio.file.Files.newOutputStream(Files.java:216)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:406)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:254)
   [junit4]   2> 	at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:667)
   [junit4]   2> 	at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44)
   [junit4]   2> 	at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
   [junit4]   2> 	at org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.<init>(Lucene80DocValuesConsumer.java:73)
   [junit4]   2> 	at org.apache.lucene.codecs.lucene80.Lucene80DocValuesFormat.fieldsConsumer(Lucene80DocValuesFormat.java:141)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:224)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:139)
   [junit4]   2> 	at org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:196)
   [junit4]   2> 	at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:151)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
   [junit4]   2> 	at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2158)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1991)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1940)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$2.reindex(TestDemoParallelLeafReader.java:746)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:405)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:163)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.wrapForMerge(TestDemoParallelLeafReader.java:533)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4417)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
   [junit4]   2> 
   [junit4]   2> ??? 16, 2019 1:22:39 ?? com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
   [junit4]   2> ??: Uncaught exception in thread: Thread[Lucene Merge Thread #70,5,TGRP-TestDemoParallelLeafReader]
   [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/lucene/build/core/test/J2/temp/lucene.index.TestDemoParallelLeafReader_D1AD2D71F762344B-001/tempDir-002/segs/4co6e0ws25r8xxi24lsus859k_122/_0_Direct_0.dvdd: Too many open files
   [junit4]   2> 	at __randomizedtesting.SeedInfo.seed([D1AD2D71F762344B]:0)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
   [junit4]   2> Caused by: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/lucene/build/core/test/J2/temp/lucene.index.TestDemoParallelLeafReader_D1AD2D71F762344B-001/tempDir-002/segs/4co6e0ws25r8xxi24lsus859k_122/_0_Direct_0.dvdd: Too many open files
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
   [junit4]   2> 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
   [junit4]   2> 	at java.nio.file.Files.newOutputStream(Files.java:216)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:406)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:254)
   [junit4]   2> 	at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:667)
   [junit4]   2> 	at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44)
   [junit4]   2> 	at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
   [junit4]   2> 	at org.apache.lucene.codecs.memory.DirectDocValuesConsumer.<init>(DirectDocValuesConsumer.java:55)
   [junit4]   2> 	at org.apache.lucene.codecs.memory.DirectDocValuesFormat.fieldsConsumer(DirectDocValuesFormat.java:71)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:224)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:160)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:109)
   [junit4]   2> 	at org.apache.lucene.index.NumericDocValuesWriter.flush(NumericDocValuesWriter.java:122)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:263)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:138)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:468)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:555)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:722)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3591)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:3566)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1960)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1940)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$2.reindex(TestDemoParallelLeafReader.java:746)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:405)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:163)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.wrapForMerge(TestDemoParallelLeafReader.java:533)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4417)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
   [junit4]   2> 
   [junit4]   2> ??? 16, 2019 1:22:39 ?? com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
   [junit4]   2> ??: Uncaught exception in thread: Thread[Lucene Merge Thread #69,5,TGRP-TestDemoParallelLeafReader]
   [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/lucene/build/core/test/J2/temp/lucene.index.TestDemoParallelLeafReader_D1AD2D71F762344B-001/tempDir-002/segs/4co6e0ws25r8xxi24lsus8585_122/_0_Direct_0.dvdm: Too many open files
   [junit4]   2> 	at __randomizedtesting.SeedInfo.seed([D1AD2D71F762344B]:0)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
   [junit4]   2> Caused by: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/lucene/build/core/test/J2/temp/lucene.index.TestDemoParallelLeafReader_D1AD2D71F762344B-001/tempDir-002/segs/4co6e0ws25r8xxi24lsus8585_122/_0_Direct_0.dvdm: Too many open files
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
   [junit4]   2> 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
   [junit4]   2> 	at java.nio.file.Files.newOutputStream(Files.java:216)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:406)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:254)
   [junit4]   2> 	at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:667)
   [junit4]   2> 	at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44)
   [junit4]   2> 	at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
   [junit4]   2> 	at org.apache.lucene.codecs.memory.DirectDocValuesConsumer.<init>(DirectDocValuesConsumer.java:58)
   [junit4]   2> 	at org.apache.lucene.codecs.memory.DirectDocValuesFormat.fieldsConsumer(DirectDocValuesFormat.java:71)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:224)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:160)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:109)
   [junit4]   2> 	at org.apache.lucene.index.NumericDocValuesWriter.flush(NumericDocValuesWriter.java:122)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:263)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:138)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:468)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:555)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:722)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3591)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:3566)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1960)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1940)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$2.reindex(TestDemoParallelLeafReader.java:746)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:405)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:163)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.wrapForMerge(TestDemoParallelLeafReader.java:533)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4417)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
   [junit4]   2> 
   [junit4]   2> ??? 16, 2019 1:22:39 ?? com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
   [junit4]   2> ??: Uncaught exception in thread: Thread[Lucene Merge Thread #68,5,TGRP-TestDemoParallelLeafReader]
   [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/lucene/build/core/test/J2/temp/lucene.index.TestDemoParallelLeafReader_D1AD2D71F762344B-001/tempDir-002/segs/4co6e0ws25r8xxi24lsus8565_122/_1_Lucene80_0.dvm: Too many open files
   [junit4]   2> 	at __randomizedtesting.SeedInfo.seed([D1AD2D71F762344B]:0)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
   [junit4]   2> Caused by: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/lucene/build/core/test/J2/temp/lucene.index.TestDemoParallelLeafReader_D1AD2D71F762344B-001/tempDir-002/segs/4co6e0ws25r8xxi24lsus8565_122/_1_Lucene80_0.dvm: Too many open files
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
   [junit4]   2> 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
   [junit4]   2> 	at java.nio.file.Files.newOutputStream(Files.java:216)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:406)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:254)
   [junit4]   2> 	at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:667)
   [junit4]   2> 	at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44)
   [junit4]   2> 	at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
   [junit4]   2> 	at org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.<init>(Lucene80DocValuesConsumer.java:73)
   [junit4]   2> 	at org.apache.lucene.codecs.lucene80.Lucene80DocValuesFormat.fieldsConsumer(Lucene80DocValuesFormat.java:141)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:224)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:139)
   [junit4]   2> 	at org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:196)
   [junit4]   2> 	at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:151)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
   [junit4]   2> 	at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2158)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1991)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1940)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$2.reindex(TestDemoParallelLeafReader.java:746)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:405)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:163)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.wrapForMerge(TestDemoParallelLeafReader.java:533)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4417)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
   [junit4]   2> 
   [junit4]   2> ??? 16, 2019 1:22:39 ?? com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
   [junit4]   2> ??: Uncaught exception in thread: Thread[Lucene Merge Thread #66,5,TGRP-TestDemoParallelLeafReader]
   [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/lucene/build/core/test/J2/temp/lucene.index.TestDemoParallelLeafReader_D1AD2D71F762344B-001/tempDir-002/segs/4co6e0ws25r8xxi24lsus853r_122/_0_Direct_0.dvdd: Too many open files
   [junit4]   2> 	at __randomizedtesting.SeedInfo.seed([D1AD2D71F762344B]:0)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
   [junit4]   2> Caused by: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/lucene/build/core/test/J2/temp/lucene.index.TestDemoParallelLeafReader_D1AD2D71F762344B-001/tempDir-002/segs/4co6e0ws25r8xxi24lsus853r_122/_0_Direct_0.dvdd: Too many open files
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
   [junit4]   2> 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
   [junit4]   2> 	at java.nio.file.Files.newOutputStream(Files.java:216)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:406)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:254)
   [junit4]   2> 	at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:667)
   [junit4]   2> 	at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44)
   [junit4]   2> 	at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
   [junit4]   2> 	at org.apache.lucene.codecs.memory.DirectDocValuesConsumer.<init>(DirectDocValuesConsumer.java:55)
   [junit4]   2> 	at org.apache.lucene.codecs.memory.DirectDocValuesFormat.fieldsConsumer(DirectDocValuesFormat.java:71)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:224)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:160)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:109)
   [junit4]   2> 	at org.apache.lucene.index.NumericDocValuesWriter.flush(NumericDocValuesWriter.java:122)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:263)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:138)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:468)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:555)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:722)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3591)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:3566)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1960)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1940)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$2.reindex(TestDemoParallelLeafReader.java:746)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:405)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:163)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.wrapForMerge(TestDemoParallelLeafReader.java:533)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4417)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
   [junit4]   2> 
   [junit4]   2> ??? 16, 2019 1:22:39 ?? com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
   [junit4]   2> ??: Uncaught exception in thread: Thread[Lucene Merge Thread #64,5,TGRP-TestDemoParallelLeafReader]
   [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/lucene/build/core/test/J2/temp/lucene.index.TestDemoParallelLeafReader_D1AD2D71F762344B-001/tempDir-002/segs/4co6e0ws25r8xxi24lsus8517_122/_1_Lucene80_0.dvm: Too many open files
   [junit4]   2> 	at __randomizedtesting.SeedInfo.seed([D1AD2D71F762344B]:0)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
   [junit4]   2> Caused by: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/lucene/build/core/test/J2/temp/lucene.index.TestDemoParallelLeafReader_D1AD2D71F762344B-001/tempDir-002/segs/4co6e0ws25r8xxi24lsus8517_122/_1_Lucene80_0.dvm: Too many open files
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
   [junit4]   2> 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
   [junit4]   2> 	at java.nio.file.Files.newOutputStream(Files.java:216)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:406)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:254)
   [junit4]   2> 	at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:667)
   [junit4]   2> 	at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44)
   [junit4]   2> 	at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
   [junit4]   2> 	at org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.<init>(Lucene80DocValuesConsumer.java:73)
   [junit4]   2> 	at org.apache.lucene.codecs.lucene80.Lucene80DocValuesFormat.fieldsConsumer(Lucene80DocValuesFormat.java:141)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:224)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:139)
   [junit4]   2> 	at org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:196)
   [junit4]   2> 	at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:151)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
   [junit4]   2> 	at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2158)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1991)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1940)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$2.reindex(TestDemoParallelLeafReader.java:746)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:405)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:163)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.wrapForMerge(TestDemoParallelLeafReader.java:533)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4417)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
   [junit4]   2> 
   [junit4]   2> ??? 16, 2019 1:22:39 ?? com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
   [junit4]   2> ??: Uncaught exception in thread: Thread[Lucene Merge Thread #65,5,TGRP-TestDemoParallelLeafReader]
   [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/lucene/build/core/test/J2/temp/lucene.index.TestDemoParallelLeafReader_D1AD2D71F762344B-001/tempDir-002/segs/4co6e0ws25r8xxi24lsus852d_122/_0_Direct_0.dvdm: Too many open files
   [junit4]   2> 	at __randomizedtesting.SeedInfo.seed([D1AD2D71F762344B]:0)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
   [junit4]   2> Caused by: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/lucene/build/core/test/J2/temp/lucene.index.TestDemoParallelLeafReader_D1AD2D71F762344B-001/tempDir-002/segs/4co6e0ws25r8xxi24lsus852d_122/_0_Direct_0.dvdm: Too many open files
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
   [junit4]   2> 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
   [junit4]   2> 	at java.nio.file.Files.newOutputStream(Files.java:216)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:406)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:254)
   [junit4]   2> 	at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:667)
   [junit4]   2> 	at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44)
   [junit4]   2> 	at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
   [junit4]   2> 	at org.apache.lucene.codecs.memory.DirectDocValuesConsumer.<init>(DirectDocValuesConsumer.java:58)
   [junit4]   2> 	at org.apache.lucene.codecs.memory.DirectDocValuesFormat.fieldsConsumer(DirectDocValuesFormat.java:71)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:224)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:160)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:109)
   [junit4]   2> 	at org.apache.lucene.index.NumericDocValuesWriter.flush(NumericDocValuesWriter.java:122)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:263)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:138)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:468)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:555)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:722)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3591)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:3566)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1960)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1940)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$2.reindex(TestDemoParallelLeafReader.java:746)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:405)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:163)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.wrapForMerge(TestDemoParallelLeafReader.java:533)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4417)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
   [junit4]   2> 
   [junit4]   2> ??? 16, 2019 1:22:39 ?? com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
   [junit4]   2> ??: Uncaught exception in thread: Thread[Lucene Merge Thread #62,5,TGRP-TestDemoParallelLeafReader]
   [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/lucene/build/core/test/J2/temp/lucene.index.TestDemoParallelLeafReader_D1AD2D71F762344B-001/tempDir-002/segs/4co6e0ws25r8xxi24lsus84y9_122/_0_Direct_0.dvdd: Too many open files
   [junit4]   2> 	at __randomizedtesting.SeedInfo.seed([D1AD2D71F762344B]:0)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
   [junit4]   2> Caused by: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/lucene/build/core/test/J2/temp/lucene.index.TestDemoParallelLeafReader_D1AD2D71F762344B-001/tempDir-002/segs/4co6e0ws25r8xxi24lsus84y9_122/_0_Direct_0.dvdd: Too many open files
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
   [junit4]   2> 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
   [junit4]   2> 	at java.nio.file.Files.newOutputStream(Files.java:216)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:406)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:254)
   [junit4]   2> 	at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:667)
   [junit4]   2> 	at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44)
   [junit4]   2> 	at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
   [junit4]   2> 	at org.apache.lucene.codecs.memory.DirectDocValuesConsumer.<init>(DirectDocValuesConsumer.java:55)
   [junit4]   2> 	at org.apache.lucene.codecs.memory.DirectDocValuesFormat.fieldsConsumer(DirectDocValuesFormat.java:71)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:224)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:160)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:109)
   [junit4]   2> 	at org.apache.lucene.index.NumericDocValuesWriter.flush(NumericDocValuesWriter.java:122)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:263)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:138)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:468)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:555)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:722)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3591)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:3566)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1960)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1940)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$2.reindex(TestDemoParallelLeafReader.java:746)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:405)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:163)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.wrapForMerge(TestDemoParallelLeafReader.java:533)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4417)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
   [junit4]   2> 
   [junit4]   2> ??? 16, 2019 1:22:39 ?? com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
   [junit4]   2> ??: Uncaught exception in thread: Thread[Lucene Merge Thread #61,5,TGRP-TestDemoParallelLeafReader]
   [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/lucene/build/core/test/J2/temp/lucene.index.TestDemoParallelLeafReader_D1AD2D71F762344B-001/tempDir-002/segs/4co6e0ws25r8xxi24lsus84vt_122/_0_Direct_0.dvdm: Too many open files
   [junit4]   2> 	at __randomizedtesting.SeedInfo.seed([D1AD2D71F762344B]:0)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
   [junit4]   2> Caused by: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/lucene/build/core/test/J2/temp/lucene.index.TestDemoParallelLeafReader_D1AD2D71F762344B-001/tempDir-002/segs/4co6e0ws25r8xxi24lsus84vt_122/_0_Direct_0.dvdm: Too many open files
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
   [junit4]   2> 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
   [junit4]   2> 	at java.nio.file.Files.newOutputStream(Files.java:216)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:406)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:254)
   [junit4]   2> 	at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:667)
   [junit4]   2> 	at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44)
   [junit4]   2> 	at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
   [junit4]   2> 	at org.apache.lucene.codecs.memory.DirectDocValuesConsumer.<init>(DirectDocValuesConsumer.java:58)
   [junit4]   2> 	at org.apache.lucene.codecs.memory.DirectDocValuesFormat.fieldsConsumer(DirectDocValuesFormat.java:71)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:224)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:160)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:109)
   [junit4]   2> 	at org.apache.lucene.index.NumericDocValuesWriter.flush(NumericDocValuesWriter.java:122)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:263)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:138)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:468)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:555)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:722)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3591)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:3566)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1960)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1940)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$2.reindex(TestDemoParallelLeafReader.java:746)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:405)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:163)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.wrapForMerge(TestDemoParallelLeafReader.java:533)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4417)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
   [junit4]   2> 
   [junit4]   2> ??? 16, 2019 1:22:39 ?? com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
   [junit4]   2> ??: Uncaught exception in thread: Thread[Lucene Merge Thread #0,5,TGRP-TestDemoParallelLeafReader]
   [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/lucene/build/core/test/J2/temp/lucene.index.TestDemoParallelLeafReader_D1AD2D71F762344B-001/tempDir-002/segs/4co6e0ws25r8xxi24lsus84v8_122/_1_Lucene80_0.dvm: Too many open files
   [junit4]   2> 	at __randomizedtesting.SeedInfo.seed([D1AD2D71F762344B]:0)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
   [junit4]   2> Caused by: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/lucene/build/core/test/J2/temp/lucene.index.TestDemoParallelLeafReader_D1AD2D71F762344B-001/tempDir-002/segs/4co6e0ws25r8xxi24lsus84v8_122/_1_Lucene80_0.dvm: Too many open files
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
   [junit4]   2> 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
   [junit4]   2> 	at java.nio.file.Files.newOutputStream(Files.java:216)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:406)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:254)
   [junit4]   2> 	at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:667)
   [junit4]   2> 	at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$1.createOutput(ConcurrentMergeScheduler.java:288)
   [junit4]   2> 	at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
   [junit4]   2> 	at org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.<init>(Lucene80DocValuesConsumer.java:73)
   [junit4]   2> 	at org.apache.lucene.codecs.lucene80.Lucene80DocValuesFormat.fieldsConsumer(Lucene80DocValuesFormat.java:141)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:224)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:139)
   [junit4]   2> 	at org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:196)
   [junit4]   2> 	at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:151)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
   [junit4]   2> 
   [junit4]   2> ??? 16, 2019 1:22:39 ?? com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
   [junit4]   2> ??: Uncaught exception in thread: Thread[Lucene Merge Thread #0,5,TGRP-TestDemoParallelLeafReader]
   [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/lucene/build/core/test/J2/temp/lucene.index.TestDemoParallelLeafReader_D1AD2D71F762344B-001/tempDir-002/segs/4co6e0ws25r8xxi24lsus84u5_122/_2_Lucene80_0.dvm: Too many open files
   [junit4]   2> 	at __randomizedtesting.SeedInfo.seed([D1AD2D71F762344B]:0)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
   [junit4]   2> Caused by: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/lucene/build/core/test/J2/temp/lucene.index.TestDemoParallelLeafReader_D1AD2D71F762344B-001/tempDir-002/segs/4co6e0ws25r8xxi24lsus84u5_122/_2_Lucene80_0.dvm: Too many open files
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
   [junit4]   2> 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
   [junit4]   2> 	at java.nio.file.Files.newOutputStream(Files.java:216)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:406)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:254)
   [junit4]   2> 	at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:667)
   [junit4]   2> 	at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$1.createOutput(ConcurrentMergeScheduler.java:288)
   [junit4]   2> 	at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
   [junit4]   2> 	at org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.<init>(Lucene80DocValuesConsumer.java:73)
   [junit4]   2> 	at org.apache.lucene.codecs.lucene80.Lucene80DocValuesFormat.fieldsConsumer(Lucene80DocValuesFormat.java:141)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:224)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:139)
   [junit4]   2> 	at org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:196)
   [junit4]   2> 	at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:151)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
   [junit4]   2> 
   [junit4]   2> ??? 16, 2019 1:22:39 ?? com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
   [junit4]   2> ??: Uncaught exception in thread: Thread[Lucene Merge Thread #59,5,TGRP-TestDemoParallelLeafReader]
   [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.lang.IllegalStateException: this writer hit an unrecoverable error; cannot complete forceMerge
   [junit4]   2> 	at __randomizedtesting.SeedInfo.seed([D1AD2D71F762344B]:0)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
   [junit4]   2> Caused by: java.lang.IllegalStateException: this writer hit an unrecoverable error; cannot complete forceMerge
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1998)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1940)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$2.reindex(TestDemoParallelLeafReader.java:746)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:405)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:163)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.wrapForMerge(TestDemoParallelLeafReader.java:533)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4417)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
   [junit4]   2> Caused by: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/lucene/build/core/test/J2/temp/lucene.index.TestDemoParallelLeafReader_D1AD2D71F762344B-001/tempDir-002/segs/4co6e0ws25r8xxi24lsus84v8_122/_1_Lucene80_0.dvm: Too many open files
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
   [junit4]   2> 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
   [junit4]   2> 	at java.nio.file.Files.newOutputStream(Files.java:216)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:406)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:254)
   [junit4]   2> 	at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:667)
   [junit4]   2> 	at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$1.createOutput(ConcurrentMergeScheduler.java:288)
   [junit4]   2> 	at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
   [junit4]   2> 	at org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.<init>(Lucene80DocValuesConsumer.java:73)
   [junit4]   2> 	at org.apache.lucene.codecs.lucene80.Lucene80DocValuesFormat.fieldsConsumer(Lucene80DocValuesFormat.java:141)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:224)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:139)
   [junit4]   2> 	at org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:196)
   [junit4]   2> 	at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:151)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
   [junit4]   2> 	... 3 more
   [junit4]   2> 
   [junit4]   2> ??? 16, 2019 1:22:39 ?? com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
   [junit4]   2> ??: Uncaught exception in thread: Thread[Lucene Merge Thread #60,5,TGRP-TestDemoParallelLeafReader]
   [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/lucene/build/core/test/J2/temp/lucene.index.TestDemoParallelLeafReader_D1AD2D71F762344B-001/tempDir-002/segs/4co6e0ws25r8xxi24lsus84vz_122/_0_Direct_0.dvdd: Too many open files
   [junit4]   2> 	at __randomizedtesting.SeedInfo.seed([D1AD2D71F762344B]:0)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
   [junit4]   2> Caused by: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/lucene/build/core/test/J2/temp/lucene.index.TestDemoParallelLeafReader_D1AD2D71F762344B-001/tempDir-002/segs/4co6e0ws25r8xxi24lsus84vz_122/_0_Direct_0.dvdd: Too many open files
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
   [junit4]   2> 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
   [junit4]   2> 	at java.nio.file.Files.newOutputStream(Files.java:216)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:406)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:254)
   [junit4]   2> 	at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:667)
   [junit4]   2> 	at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44)
   [junit4]   2> 	at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
   [junit4]   2> 	at org.apache.lucene.codecs.memory.DirectDocValuesConsumer.<init>(DirectDocValuesConsumer.java:55)
   [junit4]   2> 	at org.apache.lucene.codecs.memory.DirectDocValuesFormat.fieldsConsumer(DirectDocValuesFormat.java:71)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:224)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:160)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:109)
   [junit4]   2> 	at org.apache.lucene.index.NumericDocValuesWriter.flush(NumericDocValuesWriter.java:122)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:263)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:138)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:468)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:555)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:722)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3591)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:3566)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1960)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1940)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$2.reindex(TestDemoParallelLeafReader.java:746)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:405)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:163)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.wrapForMerge(TestDemoParallelLeafReader.java:533)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4417)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
   [junit4]   2> 
   [junit4]   2> ??? 16, 2019 1:22:39 ?? com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
   [junit4]   2> ??: Uncaught exception in thread: Thread[Lucene Merge Thread #63,5,TGRP-TestDemoParallelLeafReader]
   [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/lucene/build/core/test/J2/temp/lucene.index.TestDemoParallelLeafReader_D1AD2D71F762344B-001/tempDir-002/segs/4co6e0ws25r8xxi24lsus850j_122/_0.dim: Too many open files
   [junit4]   2> 	at __randomizedtesting.SeedInfo.seed([D1AD2D71F762344B]:0)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
   [junit4]   2> Caused by: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/lucene/build/core/test/J2/temp/lucene.index.TestDemoParallelLeafReader_D1AD2D71F762344B-001/tempDir-002/segs/4co6e0ws25r8xxi24lsus850j_122/_0.dim: Too many open files
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.newByteChannel(HandleTrackingFS.java:271)
   [junit4]   2> 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newByteChannel(FilterFileSystemProvider.java:212)
   [junit4]   2> 	at java.nio.file.Files.newByteChannel(Files.java:361)
   [junit4]   2> 	at java.nio.file.Files.newByteChannel(Files.java:407)
   [junit4]   2> 	at org.apache.lucene.store.SimpleFSDirectory.openInput(SimpleFSDirectory.java:77)
   [junit4]   2> 	at org.apache.lucene.store.NRTCachingDirectory.openInput(NRTCachingDirectory.java:181)
   [junit4]   2> 	at org.apache.lucene.util.LuceneTestCase.slowFileExists(LuceneTestCase.java:2862)
   [junit4]   2> 	at org.apache.lucene.store.MockDirectoryWrapper.openInput(MockDirectoryWrapper.java:747)
   [junit4]   2> 	at org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100)
   [junit4]   2> 	at org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100)
   [junit4]   2> 	at org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100)
   [junit4]   2> 	at org.apache.lucene.store.Directory.openChecksumInput(Directory.java:157)
   [junit4]   2> 	at org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.write(Lucene50CompoundFormat.java:89)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.createCompoundFile(IndexWriter.java:5014)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriterPerThread.sealFlushedSegment(DocumentsWriterPerThread.java:574)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:513)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:555)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:722)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3591)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:3566)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1960)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1940)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$2.reindex(TestDemoParallelLeafReader.java:746)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:405)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:163)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.wrapForMerge(TestDemoParallelLeafReader.java:533)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4417)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
   [junit4]   2> 
   [junit4]   2> ??? 16, 2019 1:22:39 ?? com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
   [junit4]   2> ??: Uncaught exception in thread: Thread[Lucene Merge Thread #56,5,TGRP-TestDemoParallelLeafReader]
   [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/lucene/build/core/test/J2/temp/lucene.index.TestDemoParallelLeafReader_D1AD2D71F762344B-001/tempDir-002/segs/4co6e0ws25r8xxi24lsus84td_122/_0.fdt: Too many open files
   [junit4]   2> 	at __randomizedtesting.SeedInfo.seed([D1AD2D71F762344B]:0)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
   [junit4]   2> Caused by: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/lucene/build/core/test/J2/temp/lucene.index.TestDemoParallelLeafReader_D1AD2D71F762344B-001/tempDir-002/segs/4co6e0ws25r8xxi24lsus84td_122/_0.fdt: Too many open files
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
   [junit4]   2> 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
   [junit4]   2> 	at java.nio.file.Files.newOutputStream(Files.java:216)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:406)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:254)
   [junit4]   2> 	at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:667)
   [junit4]   2> 	at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44)
   [junit4]   2> 	at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
   [junit4]   2> 	at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:115)
   [junit4]   2> 	at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:128)
   [junit4]   2> 	at org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:183)
   [junit4]   2> 	at org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:48)
   [junit4]   2> 	at org.apache.lucene.index.StoredFieldsConsumer.initStoredFieldsWriter(StoredFieldsConsumer.java:39)
   [junit4]   2> 	at org.apache.lucene.index.StoredFieldsConsumer.startDocument(StoredFieldsConsumer.java:46)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.startStoredFields(DefaultIndexingChain.java:367)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:403)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:250)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:495)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1594)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1213)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$2.reindex(TestDemoParallelLeafReader.java:730)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:405)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:163)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.wrapForMerge(TestDemoParallelLeafReader.java:533)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4417)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
   [junit4]   2> 
   [junit4]   2> ??? 16, 2019 1:22:39 ?? com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
   [junit4]   2> ??: Uncaught exception in thread: Thread[Lucene Merge Thread #71,5,TGRP-TestDemoParallelLeafReader]
   [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/lucene/build/core/test/J2/temp/lucene.index.TestDemoParallelLeafReader_D1AD2D71F762344B-001/tempDir-002/segs/4co6e0ws25r8xxi24lsus85ca_122/_1_Lucene80_0.dvm: Too many open files
   [junit4]   2> 	at __randomizedtesting.SeedInfo.seed([D1AD2D71F762344B]:0)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
   [junit4]   2> Caused by: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/lucene/build/core/test/J2/temp/lucene.index.TestDemoParallelLeafReader_D1AD2D71F762344B-001/tempDir-002/segs/4co6e0ws25r8xxi24lsus85ca_122/_1_Lucene80_0.dvm: Too many open files
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
   [junit4]   2> 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
   [junit4]   2> 	at java.nio.file.Files.newOutputStream(Files.java:216)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:406)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:254)
   [junit4]   2> 	at org.apache.lucene.store.NRTCachingDirectory.createOutput(NRTCachingDirectory.java:146)
   [junit4]   2> 	at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:667)
   [junit4]   2> 	at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44)
   [junit4]   2> 	at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
   [junit4]   2> 	at org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.<init>(Lucene80DocValuesConsumer.java:73)
   [junit4]   2> 	at org.apache.lucene.codecs.lucene80.Lucene80DocValuesFormat.fieldsConsumer(Lucene80DocValuesFormat.java:141)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:224)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:139)
   [junit4]   2> 	at org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:196)
   [junit4]   2> 	at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:151)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.jav

[...truncated too long message...]

cValuesFormat.java:224)
   [junit4]    > 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:139)
   [junit4]    > 	at org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:196)
   [junit4]    > 	at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:151)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
   [junit4]    > 	... 3 moreThrowable #24: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=19536, name=Lucene Merge Thread #67, state=RUNNABLE, group=TGRP-TestDemoParallelLeafReader]
   [junit4]    > Caused by: org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/lucene/build/core/test/J2/temp/lucene.index.TestDemoParallelLeafReader_D1AD2D71F762344B-001/tempDir-002/segs/4co6e0ws25r8xxi24lsus855a_122/_0_Direct_0.dvdm: Too many open files
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([D1AD2D71F762344B]:0)
   [junit4]    > 	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
   [junit4]    > 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
   [junit4]    > Caused by: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/lucene/build/core/test/J2/temp/lucene.index.TestDemoParallelLeafReader_D1AD2D71F762344B-001/tempDir-002/segs/4co6e0ws25r8xxi24lsus855a_122/_0_Direct_0.dvdm: Too many open files
   [junit4]    > 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
   [junit4]    > 	at java.nio.file.Files.newOutputStream(Files.java:216)
   [junit4]    > 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
   [junit4]    > 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:406)
   [junit4]    > 	at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:254)
   [junit4]    > 	at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:667)
   [junit4]    > 	at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44)
   [junit4]    > 	at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
   [junit4]    > 	at org.apache.lucene.codecs.memory.DirectDocValuesConsumer.<init>(DirectDocValuesConsumer.java:58)
   [junit4]    > 	at org.apache.lucene.codecs.memory.DirectDocValuesFormat.fieldsConsumer(DirectDocValuesFormat.java:71)
   [junit4]    > 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:224)
   [junit4]    > 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:160)
   [junit4]    > 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:109)
   [junit4]    > 	at org.apache.lucene.index.NumericDocValuesWriter.flush(NumericDocValuesWriter.java:122)
   [junit4]    > 	at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:263)
   [junit4]    > 	at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:138)
   [junit4]    > 	at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:468)
   [junit4]    > 	at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:555)
   [junit4]    > 	at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:722)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3591)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:3566)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1960)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1940)
   [junit4]    > 	at org.apache.lucene.index.TestDemoParallelLeafReader$2.reindex(TestDemoParallelLeafReader.java:746)
   [junit4]    > 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:405)
   [junit4]    > 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:163)
   [junit4]    > 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.wrapForMerge(TestDemoParallelLeafReader.java:533)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4417)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
   [junit4]    > 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
   [junit4]    > 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/lucene/build/core/test/J2/temp/lucene.index.TestDemoParallelLeafReader_D1AD2D71F762344B-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene84): {text=PostingsFormat(name=LuceneVarGapDocFreqInterval), id=Lucene84}, docValues:{number_56=DocValuesFormat(name=Lucene80), number_119=DocValuesFormat(name=Direct), number_33=DocValuesFormat(name=Lucene80), number_118=DocValuesFormat(name=Lucene80), number_31=DocValuesFormat(name=Asserting), number_50=DocValuesFormat(name=Direct), number_94=DocValuesFormat(name=Direct), number_9=DocValuesFormat(name=Direct), number_8=DocValuesFormat(name=Lucene80), number_15=DocValuesFormat(name=Lucene80), number_37=DocValuesFormat(name=Lucene80), number_36=DocValuesFormat(name=Direct), number_13=DocValuesFormat(name=Asserting), number=DocValuesFormat(name=Lucene80), number_122=DocValuesFormat(name=Direct), number_121=DocValuesFormat(name=Lucene80), number_120=DocValuesFormat(name=Lucene80), text=DocValuesFormat(name=Direct), id=DocValuesFormat(name=Lucene80), number_21=DocValuesFormat(name=Direct), number_42=DocValuesFormat(name=Asserting), number_29=DocValuesFormat(name=Direct), number_27=DocValuesFormat(name=Lucene80), number_46=DocValuesFormat(name=Asserting), number_3=DocValuesFormat(name=Lucene80), number_2=DocValuesFormat(name=Asserting), number_1=DocValuesFormat(name=Direct), number_0=DocValuesFormat(name=Lucene80)}, maxPointsInLeafNode=1612, maxMBSortInHeap=7.583878764501662, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@d6e79d9), locale=zh-TW, timezone=America/Indiana/Knox
   [junit4]   2> NOTE: Linux 4.15.0-54-generic amd64/Oracle Corporation 1.8.0_191 (64-bit)/cpus=4,threads=1,free=111936920,total=409468928
   [junit4]   2> NOTE: All tests run in this JVM: [TestIndexingSequenceNumbers, TestExternalCodecs, TestBooleanQuery, TestSubScorerFreqs, TestFieldUpdatesBuffer, TestTwoPhaseCommitTool, Test2BTerms, TestFieldInfos, TestDisjunctionMaxQuery, Test2BPostingsBytes, TestBytesRefAttImpl, TestFloatRangeFieldQueries, TestBoolean2ScorerSupplier, TestParallelLeafReader, TestCharsRefBuilder, TestReaderWrapperDVTypeCheck, TestLongPostings, TestPrefixInBooleanQuery, TestGeoUtils, TestOmitNorms, TestCustomTermFreq, TestNoMergeScheduler, TestPackedInts, TestBlendedTermQuery, TestBinaryDocument, TestToken, TestIsCurrent, TestSPIClassIterator, TestIndexTooManyDocs, TestLongRangeFieldQueries, TestSpanFirstQuery, TestCharacterUtils, TestDirectory, TestPerFieldDocValuesFormat, TestScoreCachingWrappingScorer, Test2BBKDPoints, TestPendingSoftDeletes, TestLatLonDocValuesQueries, TestStressNRT, TestLucene50TermVectorsFormat, TestBufferedChecksum, TestDocsAndPositions, TestTopDocsCollector, TestMultiLevelSkipList, TestOperations, TestFieldCacheRewriteMethod, TestHighCompressionMode, TestFutureArrays, TestSloppyMath, TestByteSlices, TestNot, TestIndexWriterOutOfFileDescriptors, TestGraphTokenizers, TestRecyclingIntBlockAllocator, Test2BPagedBytes, TestMathUtil, TestGrowableByteArrayDataOutput, TestUsageTrackingFilterCachingPolicy, TestSpanNotQuery, TestBKDRadixSort, TestBasicModelIne, TestMultiTermQueryRewrites, TestFixedBitSet, TestNRTCachingDirectory, TestFstDirectAddressing, TestAllFilesHaveCodecHeader, TestFeatureSort, TestSimpleSearchEquivalence, TestCompiledAutomaton, TestVersion, TestNRTReaderWithThreads, TestSparseFixedBitSet, TestIndexWriterConfig, TestRadixSelector, TestDoubleValuesSource, TestConjunctions, TestApproximationSearchEquivalence, TestIndexWriterExceptions, TestThreadedForceMerge, TestLMJelinekMercerSimilarity, TestDistributionLL, TestMixedDocValuesUpdates, TestDeterminizeLexicon, TestFloatRange, TestBasicModelG, TestIndexWriterFromReader, TestLucene80NormsFormat, TestLucene80NormsFormatMergeInstance, TestForDeltaUtil, TestForUtil, TestDateTools, TestDocument, TestDoubleRange, TestIntRange, TestLatLonDocValuesField, TestLatLonPoint, TestLatLonPointDistanceFeatureQuery, TestLatLonPointDistanceSort, TestLongDistanceFeatureQuery, TestPolygon, TestPolygon2D, Test2BBinaryDocValues, Test2BDocs, Test2BNumericDocValues, Test2BPoints, Test4GBStoredFields, TestCheckIndex, TestCodecHoldsOpenFiles, TestCodecs, TestConcurrentMergeScheduler, TestConsistentFieldNumbers, TestCrash, TestDeletionPolicy, TestDemoParallelLeafReader]
   [junit4] Completed [325/528 (1!)] on J2 in 113.76s, 5 tests, 1 error <<< FAILURES!

[...truncated 68094 lines...]

[JENKINS] Lucene-Solr-NightlyTests-8.x - Build # 296 - Failure

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-8.x/296/

4 tests failed.
FAILED:  org.apache.lucene.document.TestXYMultiPolygonShapeQueries.testRandomBig

Error Message:
Java heap space

Stack Trace:
java.lang.OutOfMemoryError: Java heap space
	at __randomizedtesting.SeedInfo.seed([62453FFBE508E403:E512427474519883]:0)
	at org.apache.lucene.store.RAMFile.newBuffer(RAMFile.java:84)
	at org.apache.lucene.store.RAMFile.addBuffer(RAMFile.java:57)
	at org.apache.lucene.store.RAMOutputStream.switchCurrentBuffer(RAMOutputStream.java:168)
	at org.apache.lucene.store.RAMOutputStream.writeBytes(RAMOutputStream.java:154)
	at org.apache.lucene.store.MockIndexOutputWrapper.writeBytes(MockIndexOutputWrapper.java:141)
	at org.apache.lucene.util.bkd.OfflinePointWriter.append(OfflinePointWriter.java:56)
	at org.apache.lucene.util.bkd.BKDWriter.add(BKDWriter.java:247)
	at org.apache.lucene.index.RandomCodec$1$1$1.visit(RandomCodec.java:129)
	at org.apache.lucene.codecs.PointsWriter$1$1$1.visit(PointsWriter.java:117)
	at org.apache.lucene.index.AssertingLeafReader$AssertingIntersectVisitor.visit(AssertingLeafReader.java:1171)
	at org.apache.lucene.util.bkd.BKDReader.visitCompressedDocValues(BKDReader.java:735)
	at org.apache.lucene.util.bkd.BKDReader.visitDocValuesWithCardinality(BKDReader.java:684)
	at org.apache.lucene.util.bkd.BKDReader.visitDocValues(BKDReader.java:590)
	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:790)
	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:823)
	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:823)
	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:813)
	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:813)
	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:813)
	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:813)
	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:823)
	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:823)
	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:823)
	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:823)
	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:813)
	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:506)
	at org.apache.lucene.index.AssertingLeafReader$AssertingPointValues.intersect(AssertingLeafReader.java:1052)
	at org.apache.lucene.codecs.PointsWriter$1$1.intersect(PointsWriter.java:105)
	at org.apache.lucene.index.RandomCodec$1$1.writeField(RandomCodec.java:122)
	at org.apache.lucene.codecs.PointsWriter.mergeOneField(PointsWriter.java:62)
	at org.apache.lucene.codecs.PointsWriter.merge(PointsWriter.java:191)
	at org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.merge(Lucene60PointsWriter.java:143)


FAILED:  org.apache.lucene.document.TestXYPolygonShapeQueries.testRandomBig

Error Message:
Java heap space

Stack Trace:
java.lang.OutOfMemoryError: Java heap space
	at __randomizedtesting.SeedInfo.seed([62453FFBE508E403:E512427474519883]:0)
	at org.apache.lucene.store.RAMFile.newBuffer(RAMFile.java:84)
	at org.apache.lucene.store.RAMFile.addBuffer(RAMFile.java:57)
	at org.apache.lucene.store.RAMOutputStream.switchCurrentBuffer(RAMOutputStream.java:168)
	at org.apache.lucene.store.RAMOutputStream.writeBytes(RAMOutputStream.java:154)
	at org.apache.lucene.store.MockIndexOutputWrapper.writeBytes(MockIndexOutputWrapper.java:141)
	at org.apache.lucene.util.bkd.OfflinePointWriter.append(OfflinePointWriter.java:67)
	at org.apache.lucene.util.bkd.BKDRadixSelector.offlinePartition(BKDRadixSelector.java:282)
	at org.apache.lucene.util.bkd.BKDRadixSelector.buildHistogramAndPartition(BKDRadixSelector.java:258)
	at org.apache.lucene.util.bkd.BKDRadixSelector.buildHistogramAndPartition(BKDRadixSelector.java:267)
	at org.apache.lucene.util.bkd.BKDRadixSelector.select(BKDRadixSelector.java:126)
	at org.apache.lucene.util.bkd.BKDWriter.build(BKDWriter.java:1647)
	at org.apache.lucene.util.bkd.BKDWriter.build(BKDWriter.java:1664)
	at org.apache.lucene.util.bkd.BKDWriter.finish(BKDWriter.java:797)
	at org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.writeField(Lucene60PointsWriter.java:129)
	at org.apache.lucene.codecs.PointsWriter.mergeOneField(PointsWriter.java:62)
	at org.apache.lucene.codecs.PointsWriter.merge(PointsWriter.java:191)
	at org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.merge(Lucene60PointsWriter.java:143)
	at org.apache.lucene.index.SegmentMerger.mergePoints(SegmentMerger.java:202)
	at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:162)
	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
	at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
	at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2158)
	at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5120)
	at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1597)
	at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1213)
	at org.apache.lucene.document.BaseShapeTestCase.indexRandomShapes(BaseShapeTestCase.java:231)
	at org.apache.lucene.document.BaseShapeTestCase.verify(BaseShapeTestCase.java:212)
	at org.apache.lucene.document.BaseShapeTestCase.doTestRandom(BaseShapeTestCase.java:138)
	at org.apache.lucene.document.TestXYPolygonShapeQueries.testRandomBig(TestXYPolygonShapeQueries.java:119)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)


FAILED:  junit.framework.TestSuite.org.apache.solr.analytics.facet.RangeFacetTest

Error Message:
Error from server at http://127.0.0.1:41575/solr/collection1: Task queue processing has stalled for 20057 ms with 0 remaining elements to process.

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:41575/solr/collection1: Task queue processing has stalled for 20057 ms with 0 remaining elements to process.
	at __randomizedtesting.SeedInfo.seed([81932C87837F5555]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:665)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:265)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:368)
	at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:296)
	at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.sendRequest(BaseCloudSolrClient.java:1143)
	at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.requestWithRetryOnStaleState(BaseCloudSolrClient.java:906)
	at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.request(BaseCloudSolrClient.java:838)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
	at org.apache.solr.client.solrj.request.UpdateRequest.commit(UpdateRequest.java:240)
	at org.apache.solr.analytics.SolrAnalyticsTestCase.setupCollection(SolrAnalyticsTestCase.java:68)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:878)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.update.processor.DimensionalRoutedAliasUpdateProcessorTest.testCatTime

Error Message:
took over 10 seconds after collection creation to update aliases

Stack Trace:
java.lang.AssertionError: took over 10 seconds after collection creation to update aliases
	at __randomizedtesting.SeedInfo.seed([94A90A26848C56A1:93A24525BB7F0749]:0)
	at org.junit.Assert.fail(Assert.java:88)
	at org.apache.solr.update.processor.RoutedAliasUpdateProcessorTest.waitColAndAlias(RoutedAliasUpdateProcessorTest.java:77)
	at org.apache.solr.update.processor.DimensionalRoutedAliasUpdateProcessorTest.testCatTime(DimensionalRoutedAliasUpdateProcessorTest.java:481)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 11319 lines...]
   [junit4] Suite: org.apache.lucene.document.TestXYPolygonShapeQueries
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestXYPolygonShapeQueries -Dtests.method=testRandomBig -Dtests.seed=62453FFBE508E403 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/test-data/enwiki.random.lines.txt -Dtests.locale=hu-HU -Dtests.timezone=America/Indiana/Vevay -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR    568s J0 | TestXYPolygonShapeQueries.testRandomBig <<<
   [junit4]    > Throwable #1: java.lang.OutOfMemoryError: Java heap space
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([62453FFBE508E403:E512427474519883]:0)
   [junit4]    > 	at org.apache.lucene.store.RAMFile.newBuffer(RAMFile.java:84)
   [junit4]    > 	at org.apache.lucene.store.RAMFile.addBuffer(RAMFile.java:57)
   [junit4]    > 	at org.apache.lucene.store.RAMOutputStream.switchCurrentBuffer(RAMOutputStream.java:168)
   [junit4]    > 	at org.apache.lucene.store.RAMOutputStream.writeBytes(RAMOutputStream.java:154)
   [junit4]    > 	at org.apache.lucene.store.MockIndexOutputWrapper.writeBytes(MockIndexOutputWrapper.java:141)
   [junit4]    > 	at org.apache.lucene.util.bkd.OfflinePointWriter.append(OfflinePointWriter.java:67)
   [junit4]    > 	at org.apache.lucene.util.bkd.BKDRadixSelector.offlinePartition(BKDRadixSelector.java:282)
   [junit4]    > 	at org.apache.lucene.util.bkd.BKDRadixSelector.buildHistogramAndPartition(BKDRadixSelector.java:258)
   [junit4]    > 	at org.apache.lucene.util.bkd.BKDRadixSelector.buildHistogramAndPartition(BKDRadixSelector.java:267)
   [junit4]    > 	at org.apache.lucene.util.bkd.BKDRadixSelector.select(BKDRadixSelector.java:126)
   [junit4]    > 	at org.apache.lucene.util.bkd.BKDWriter.build(BKDWriter.java:1647)
   [junit4]    > 	at org.apache.lucene.util.bkd.BKDWriter.build(BKDWriter.java:1664)
   [junit4]    > 	at org.apache.lucene.util.bkd.BKDWriter.finish(BKDWriter.java:797)
   [junit4]    > 	at org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.writeField(Lucene60PointsWriter.java:129)
   [junit4]    > 	at org.apache.lucene.codecs.PointsWriter.mergeOneField(PointsWriter.java:62)
   [junit4]    > 	at org.apache.lucene.codecs.PointsWriter.merge(PointsWriter.java:191)
   [junit4]    > 	at org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.merge(Lucene60PointsWriter.java:143)
   [junit4]    > 	at org.apache.lucene.index.SegmentMerger.mergePoints(SegmentMerger.java:202)
   [junit4]    > 	at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:162)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
   [junit4]    > 	at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2158)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5120)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1597)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1213)
   [junit4]    > 	at org.apache.lucene.document.BaseShapeTestCase.indexRandomShapes(BaseShapeTestCase.java:231)
   [junit4]    > 	at org.apache.lucene.document.BaseShapeTestCase.verify(BaseShapeTestCase.java:212)
   [junit4]    > 	at org.apache.lucene.document.BaseShapeTestCase.doTestRandom(BaseShapeTestCase.java:138)
   [junit4]    > 	at org.apache.lucene.document.TestXYPolygonShapeQueries.testRandomBig(TestXYPolygonShapeQueries.java:119)
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/lucene/build/sandbox/test/J0/temp/lucene.document.TestXYPolygonShapeQueries_62453FFBE508E403-001
   [junit4]   2> NOTE: test params are: codec=Lucene84, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@42066a7c), locale=hu-HU, timezone=America/Indiana/Vevay
   [junit4]   2> NOTE: Linux 4.4.0-112-generic amd64/Oracle Corporation 1.8.0_191 (64-bit)/cpus=4,threads=1,free=291761336,total=371195904
   [junit4]   2> NOTE: All tests run in this JVM: [TestLatLonPointShapeQueries, TestIDVersionPostingsFormat, TestTessellator, TestXYPolygonShapeQueries]
   [junit4] Completed [23/40 (1!)] on J0 in 803.10s, 5 tests, 1 error <<< FAILURES!

[...truncated 59 lines...]
   [junit4] JVM J0: stdout was not empty, see: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/lucene/build/sandbox/test/temp/junit4-J0-20191214_150747_0433081371174697730520.sysout
   [junit4] >>> JVM J0 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: Java heap space
   [junit4] Dumping heap to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/heapdumps/java_pid18367.hprof ...
   [junit4] Heap dump file created [429542814 bytes in 0.554 secs]
   [junit4] <<< JVM J0: EOF ----

[...truncated 14 lines...]
   [junit4] Suite: org.apache.lucene.document.TestXYMultiPolygonShapeQueries
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestXYMultiPolygonShapeQueries -Dtests.method=testRandomBig -Dtests.seed=62453FFBE508E403 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/test-data/enwiki.random.lines.txt -Dtests.locale=en-IE -Dtests.timezone=Chile/EasterIsland -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   1133s J2 | TestXYMultiPolygonShapeQueries.testRandomBig <<<
   [junit4]    > Throwable #1: java.lang.OutOfMemoryError: Java heap space
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([62453FFBE508E403:E512427474519883]:0)
   [junit4]    > 	at org.apache.lucene.store.RAMFile.newBuffer(RAMFile.java:84)
   [junit4]    > 	at org.apache.lucene.store.RAMFile.addBuffer(RAMFile.java:57)
   [junit4]    > 	at org.apache.lucene.store.RAMOutputStream.switchCurrentBuffer(RAMOutputStream.java:168)
   [junit4]    > 	at org.apache.lucene.store.RAMOutputStream.writeBytes(RAMOutputStream.java:154)
   [junit4]    > 	at org.apache.lucene.store.MockIndexOutputWrapper.writeBytes(MockIndexOutputWrapper.java:141)
   [junit4]    > 	at org.apache.lucene.util.bkd.OfflinePointWriter.append(OfflinePointWriter.java:56)
   [junit4]    > 	at org.apache.lucene.util.bkd.BKDWriter.add(BKDWriter.java:247)
   [junit4]    > 	at org.apache.lucene.index.RandomCodec$1$1$1.visit(RandomCodec.java:129)
   [junit4]    > 	at org.apache.lucene.codecs.PointsWriter$1$1$1.visit(PointsWriter.java:117)
   [junit4]    > 	at org.apache.lucene.index.AssertingLeafReader$AssertingIntersectVisitor.visit(AssertingLeafReader.java:1171)
   [junit4]    > 	at org.apache.lucene.util.bkd.BKDReader.visitCompressedDocValues(BKDReader.java:735)
   [junit4]    > 	at org.apache.lucene.util.bkd.BKDReader.visitDocValuesWithCardinality(BKDReader.java:684)
   [junit4]    > 	at org.apache.lucene.util.bkd.BKDReader.visitDocValues(BKDReader.java:590)
   [junit4]    > 	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:790)
   [junit4]    > 	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:823)
   [junit4]    > 	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:823)
   [junit4]    > 	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:813)
   [junit4]    > 	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:813)
   [junit4]    > 	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:813)
   [junit4]    > 	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:813)
   [junit4]    > 	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:823)
   [junit4]    > 	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:823)
   [junit4]    > 	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:823)
   [junit4]    > 	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:823)
   [junit4]    > 	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:813)
   [junit4]    > 	at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:506)
   [junit4]    > 	at org.apache.lucene.index.AssertingLeafReader$AssertingPointValues.intersect(AssertingLeafReader.java:1052)
   [junit4]    > 	at org.apache.lucene.codecs.PointsWriter$1$1.intersect(PointsWriter.java:105)
   [junit4]    > 	at org.apache.lucene.index.RandomCodec$1$1.writeField(RandomCodec.java:122)
   [junit4]    > 	at org.apache.lucene.codecs.PointsWriter.mergeOneField(PointsWriter.java:62)
   [junit4]    > 	at org.apache.lucene.codecs.PointsWriter.merge(PointsWriter.java:191)
   [junit4]    > 	at org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.merge(Lucene60PointsWriter.java:143)
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/lucene/build/sandbox/test/J2/temp/lucene.document.TestXYMultiPolygonShapeQueries_62453FFBE508E403-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene84): {id=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene84))}, docValues:{shape=DocValuesFormat(name=Asserting), id=DocValuesFormat(name=Lucene80)}, maxPointsInLeafNode=354, maxMBSortInHeap=5.72812499855578, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@1868ec39), locale=en-IE, timezone=Chile/EasterIsland
   [junit4]   2> NOTE: Linux 4.4.0-112-generic amd64/Oracle Corporation 1.8.0_191 (64-bit)/cpus=4,threads=1,free=283552208,total=466092032
   [junit4]   2> NOTE: All tests run in this JVM: [TestPayloadSpanUtil, TestLatLonShape, TestFieldCacheTermsFilter, TestPhraseWildcardQuery, TestDoubleRangeField, TestLatLonLineShapeQueries, TestXYMultiLineShapeQueries, TestLatLonMultiLineShapeQueries, TestLatLonShapeEncoding, TestXYMultiPointShapeQueries, TestXYMultiPolygonShapeQueries]
   [junit4] Completed [40/40 (2!)] on J2 in 1457.79s, 5 tests, 1 error <<< FAILURES!

[...truncated 1 lines...]
   [junit4] JVM J2: stdout was not empty, see: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/lucene/build/sandbox/test/temp/junit4-J2-20191214_150747_0447195779847703658815.sysout
   [junit4] >>> JVM J2 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: Java heap space
   [junit4] Dumping heap to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/heapdumps/java_pid18362.hprof ...
   [junit4] Heap dump file created [461728016 bytes in 0.923 secs]
   [junit4] <<< JVM J2: EOF ----

[...truncated 5711 lines...]
   [junit4] Suite: org.apache.solr.update.processor.DimensionalRoutedAliasUpdateProcessorTest
   [junit4]   2> 8993232 INFO  (SUITE-DimensionalRoutedAliasUpdateProcessorTest-seed#[94A90A26848C56A1]-worker) [     ] o.a.s.SolrTestCaseJ4 Created dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.update.processor.DimensionalRoutedAliasUpdateProcessorTest_94A90A26848C56A1-001/data-dir-273-001
   [junit4]   2> 8993233 WARN  (SUITE-DimensionalRoutedAliasUpdateProcessorTest-seed#[94A90A26848C56A1]-worker) [     ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
   [junit4]   2> 8993233 INFO  (SUITE-DimensionalRoutedAliasUpdateProcessorTest-seed#[94A90A26848C56A1]-worker) [     ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 8993246 INFO  (SUITE-DimensionalRoutedAliasUpdateProcessorTest-seed#[94A90A26848C56A1]-worker) [     ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 8993247 INFO  (SUITE-DimensionalRoutedAliasUpdateProcessorTest-seed#[94A90A26848C56A1]-worker) [     ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 8993269 INFO  (TEST-DimensionalRoutedAliasUpdateProcessorTest.testCatTime-seed#[94A90A26848C56A1]) [     ] o.a.s.SolrTestCaseJ4 ###Starting testCatTime
   [junit4]   2> 8993269 INFO  (TEST-DimensionalRoutedAliasUpdateProcessorTest.testCatTime-seed#[94A90A26848C56A1]) [     ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 4 servers in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.update.processor.DimensionalRoutedAliasUpdateProcessorTest_94A90A26848C56A1-001/tempDir-001
   [junit4]   2> 8993270 INFO  (TEST-DimensionalRoutedAliasUpdateProcessorTest.testCatTime-seed#[94A90A26848C56A1]) [     ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 8993270 INFO  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 8993270 INFO  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 8993375 INFO  (TEST-DimensionalRoutedAliasUpdateProcessorTest.testCatTime-seed#[94A90A26848C56A1]) [     ] o.a.s.c.ZkTestServer start zk server on port:45944
   [junit4]   2> 8993376 INFO  (TEST-DimensionalRoutedAliasUpdateProcessorTest.testCatTime-seed#[94A90A26848C56A1]) [     ] o.a.s.c.ZkTestServer waitForServerUp: 127.0.0.1:45944
   [junit4]   2> 8993376 INFO  (TEST-DimensionalRoutedAliasUpdateProcessorTest.testCatTime-seed#[94A90A26848C56A1]) [     ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:45944
   [junit4]   2> 8993376 INFO  (TEST-DimensionalRoutedAliasUpdateProcessorTest.testCatTime-seed#[94A90A26848C56A1]) [     ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 45944
   [junit4]   2> 8993404 INFO  (TEST-DimensionalRoutedAliasUpdateProcessorTest.testCatTime-seed#[94A90A26848C56A1]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 8993420 INFO  (zkConnectionManagerCallback-71309-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 8993420 INFO  (TEST-DimensionalRoutedAliasUpdateProcessorTest.testCatTime-seed#[94A90A26848C56A1]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 8993432 INFO  (TEST-DimensionalRoutedAliasUpdateProcessorTest.testCatTime-seed#[94A90A26848C56A1]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 8993432 INFO  (zkConnectionManagerCallback-71311-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 8993433 INFO  (TEST-DimensionalRoutedAliasUpdateProcessorTest.testCatTime-seed#[94A90A26848C56A1]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 8993433 INFO  (TEST-DimensionalRoutedAliasUpdateProcessorTest.testCatTime-seed#[94A90A26848C56A1]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 8993434 INFO  (zkConnectionManagerCallback-71313-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 8993434 INFO  (TEST-DimensionalRoutedAliasUpdateProcessorTest.testCatTime-seed#[94A90A26848C56A1]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 8993538 WARN  (jetty-launcher-71314-thread-2) [     ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 8993538 WARN  (jetty-launcher-71314-thread-3) [     ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 8993539 WARN  (jetty-launcher-71314-thread-2) [     ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
   [junit4]   2> 8993539 INFO  (jetty-launcher-71314-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 8993539 INFO  (jetty-launcher-71314-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 8993555 WARN  (jetty-launcher-71314-thread-3) [     ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
   [junit4]   2> 8993555 INFO  (jetty-launcher-71314-thread-3) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 8993555 INFO  (jetty-launcher-71314-thread-3) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 8993555 WARN  (jetty-launcher-71314-thread-4) [     ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 8993556 WARN  (jetty-launcher-71314-thread-4) [     ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
   [junit4]   2> 8993556 INFO  (jetty-launcher-71314-thread-4) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 8993556 INFO  (jetty-launcher-71314-thread-4) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 8993556 INFO  (jetty-launcher-71314-thread-2) [     ] o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z; git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 1.8.0_191-b12
   [junit4]   2> 8993556 WARN  (jetty-launcher-71314-thread-1) [     ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 8993556 WARN  (jetty-launcher-71314-thread-1) [     ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
   [junit4]   2> 8993556 INFO  (jetty-launcher-71314-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 8993556 INFO  (jetty-launcher-71314-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 8993557 INFO  (jetty-launcher-71314-thread-1) [     ] o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z; git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 1.8.0_191-b12
   [junit4]   2> 8993557 INFO  (jetty-launcher-71314-thread-4) [     ] o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z; git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 1.8.0_191-b12
   [junit4]   2> 8993557 INFO  (jetty-launcher-71314-thread-3) [     ] o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z; git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 1.8.0_191-b12
   [junit4]   2> 8993560 INFO  (jetty-launcher-71314-thread-2) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 8993560 INFO  (jetty-launcher-71314-thread-2) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 8993560 INFO  (jetty-launcher-71314-thread-2) [     ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 8993562 INFO  (jetty-launcher-71314-thread-1) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 8993562 INFO  (jetty-launcher-71314-thread-1) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 8993562 INFO  (jetty-launcher-71314-thread-1) [     ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 8993562 INFO  (jetty-launcher-71314-thread-4) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 8993562 INFO  (jetty-launcher-71314-thread-4) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 8993574 INFO  (jetty-launcher-71314-thread-4) [     ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 8993575 INFO  (jetty-launcher-71314-thread-2) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5103cee9{/solr,null,AVAILABLE}
   [junit4]   2> 8993575 INFO  (jetty-launcher-71314-thread-1) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@71ae7365{/solr,null,AVAILABLE}
   [junit4]   2> 8993575 INFO  (jetty-launcher-71314-thread-4) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4bc28059{/solr,null,AVAILABLE}
   [junit4]   2> 8993586 INFO  (jetty-launcher-71314-thread-3) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 8993586 INFO  (jetty-launcher-71314-thread-3) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 8993586 INFO  (jetty-launcher-71314-thread-3) [     ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 8993586 INFO  (jetty-launcher-71314-thread-3) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@c736cf7{/solr,null,AVAILABLE}
   [junit4]   2> 8993592 INFO  (jetty-launcher-71314-thread-2) [     ] o.e.j.s.AbstractConnector Started ServerConnector@b791b5d{SSL,[ssl, http/1.1]}{127.0.0.1:45951}
   [junit4]   2> 8993592 INFO  (jetty-launcher-71314-thread-2) [     ] o.e.j.s.Server Started @8993671ms
   [junit4]   2> 8993592 INFO  (jetty-launcher-71314-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=45951}
   [junit4]   2> 8993592 ERROR (jetty-launcher-71314-thread-2) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 8993592 INFO  (jetty-launcher-71314-thread-2) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 8993592 INFO  (jetty-launcher-71314-thread-2) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.5.0
   [junit4]   2> 8993592 INFO  (jetty-launcher-71314-thread-2) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 8993592 INFO  (jetty-launcher-71314-thread-2) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 8993592 INFO  (jetty-launcher-71314-thread-2) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-12-14T18:51:49.999Z
   [junit4]   2> 8993594 INFO  (jetty-launcher-71314-thread-2) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 8993594 INFO  (zkConnectionManagerCallback-71316-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 8993594 INFO  (jetty-launcher-71314-thread-2) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 8993595 INFO  (jetty-launcher-71314-thread-2) [     ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 8993599 INFO  (jetty-launcher-71314-thread-3) [     ] o.e.j.s.AbstractConnector Started ServerConnector@6a8f52be{SSL,[ssl, http/1.1]}{127.0.0.1:38619}
   [junit4]   2> 8993599 INFO  (jetty-launcher-71314-thread-3) [     ] o.e.j.s.Server Started @8993679ms
   [junit4]   2> 8993599 INFO  (jetty-launcher-71314-thread-3) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=38619}
   [junit4]   2> 8993599 ERROR (jetty-launcher-71314-thread-3) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 8993599 INFO  (jetty-launcher-71314-thread-3) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 8993599 INFO  (jetty-launcher-71314-thread-3) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.5.0
   [junit4]   2> 8993599 INFO  (jetty-launcher-71314-thread-3) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 8993599 INFO  (jetty-launcher-71314-thread-3) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 8993599 INFO  (jetty-launcher-71314-thread-3) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-12-14T18:51:50.006Z
   [junit4]   2> 8993600 INFO  (jetty-launcher-71314-thread-3) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 8993600 INFO  (jetty-launcher-71314-thread-4) [     ] o.e.j.s.AbstractConnector Started ServerConnector@ffd0dbe{SSL,[ssl, http/1.1]}{127.0.0.1:34283}
   [junit4]   2> 8993600 INFO  (jetty-launcher-71314-thread-4) [     ] o.e.j.s.Server Started @8993680ms
   [junit4]   2> 8993600 INFO  (jetty-launcher-71314-thread-4) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=34283}
   [junit4]   2> 8993600 ERROR (jetty-launcher-71314-thread-4) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 8993600 INFO  (jetty-launcher-71314-thread-4) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 8993600 INFO  (jetty-launcher-71314-thread-4) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.5.0
   [junit4]   2> 8993600 INFO  (jetty-launcher-71314-thread-4) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 8993600 INFO  (jetty-launcher-71314-thread-4) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 8993600 INFO  (jetty-launcher-71314-thread-4) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-12-14T18:51:50.007Z
   [junit4]   2> 8993601 INFO  (jetty-launcher-71314-thread-4) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 8993602 INFO  (jetty-launcher-71314-thread-1) [     ] o.e.j.s.AbstractConnector Started ServerConnector@2ff48de{SSL,[ssl, http/1.1]}{127.0.0.1:35222}
   [junit4]   2> 8993602 INFO  (jetty-launcher-71314-thread-1) [     ] o.e.j.s.Server Started @8993682ms
   [junit4]   2> 8993602 INFO  (jetty-launcher-71314-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=35222}
   [junit4]   2> 8993602 ERROR (jetty-launcher-71314-thread-1) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 8993602 INFO  (jetty-launcher-71314-thread-1) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 8993602 INFO  (jetty-launcher-71314-thread-1) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.5.0
   [junit4]   2> 8993602 INFO  (jetty-launcher-71314-thread-1) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 8993602 INFO  (jetty-launcher-71314-thread-1) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 8993602 INFO  (jetty-launcher-71314-thread-1) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-12-14T18:51:50.009Z
   [junit4]   2> 8993607 INFO  (jetty-launcher-71314-thread-1) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 8993623 INFO  (zkConnectionManagerCallback-71318-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 8993627 INFO  (zkConnectionManagerCallback-71322-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 8993627 INFO  (jetty-launcher-71314-thread-1) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 8993628 INFO  (jetty-launcher-71314-thread-1) [     ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 8993632 INFO  (jetty-launcher-71314-thread-3) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 8993632 INFO  (jetty-launcher-71314-thread-3) [     ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 8993688 INFO  (zkConnectionManagerCallback-71320-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 8993688 INFO  (jetty-launcher-71314-thread-4) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 8993745 INFO  (jetty-launcher-71314-thread-4) [     ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 8994406 INFO  (jetty-launcher-71314-thread-1) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
   [junit4]   2> 8994424 WARN  (jetty-launcher-71314-thread-1) [     ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
   [junit4]   2> 8994424 INFO  (jetty-launcher-71314-thread-3) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
   [junit4]   2> 8994425 WARN  (jetty-launcher-71314-thread-3) [     ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
   [junit4]   2> 8994448 WARN  (jetty-launcher-71314-thread-1) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@2c3f0077[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 8994448 WARN  (jetty-launcher-71314-thread-1) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@2c3f0077[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 8994450 WARN  (jetty-launcher-71314-thread-3) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@594218e8[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 8994450 WARN  (jetty-launcher-71314-thread-3) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@594218e8[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 8994451 WARN  (jetty-launcher-71314-thread-1) [     ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
   [junit4]   2> 8994451 WARN  (jetty-launcher-71314-thread-1) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@55511e37[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 8994451 WARN  (jetty-launcher-71314-thread-1) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@55511e37[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 8994452 INFO  (jetty-launcher-71314-thread-1) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45944/solr
   [junit4]   2> 8994464 WARN  (jetty-launcher-71314-thread-3) [     ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
   [junit4]   2> 8994465 INFO  (jetty-launcher-71314-thread-1) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 8994465 WARN  (jetty-launcher-71314-thread-3) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@3aa88b56[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 8994465 WARN  (jetty-launcher-71314-thread-3) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@3aa88b56[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 8994466 INFO  (jetty-launcher-71314-thread-3) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45944/solr
   [junit4]   2> 8994467 INFO  (jetty-launcher-71314-thread-3) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 8994481 INFO  (zkConnectionManagerCallback-71334-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 8994481 INFO  (jetty-launcher-71314-thread-1) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 8994481 INFO  (zkConnectionManagerCallback-71338-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 8994484 INFO  (jetty-launcher-71314-thread-3) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 8994601 INFO  (jetty-launcher-71314-thread-2) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
   [junit4]   2> 8994601 WARN  (jetty-launcher-71314-thread-2) [     ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
   [junit4]   2> 8994615 WARN  (jetty-launcher-71314-thread-2) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@1dec8a5d[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 8994615 WARN  (jetty-launcher-71314-thread-2) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@1dec8a5d[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 8994616 INFO  (jetty-launcher-71314-thread-1) [n:127.0.0.1:35222_solr     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 8994617 INFO  (zkConnectionManagerCallback-71341-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 8994617 INFO  (jetty-launcher-71314-thread-1) [n:127.0.0.1:35222_solr     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 8994618 WARN  (jetty-launcher-71314-thread-2) [     ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
   [junit4]   2> 8994635 WARN  (jetty-launcher-71314-thread-2) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@58f3d260[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 8994635 WARN  (jetty-launcher-71314-thread-2) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@58f3d260[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 8994636 INFO  (jetty-launcher-71314-thread-2) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45944/solr
   [junit4]   2> 8994651 INFO  (jetty-launcher-71314-thread-2) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 8994672 INFO  (zkConnectionManagerCallback-71346-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 8994672 INFO  (jetty-launcher-71314-thread-2) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 8994725 INFO  (jetty-launcher-71314-thread-3) [n:127.0.0.1:38619_solr     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 8994760 INFO  (zkConnectionManagerCallback-71350-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 8994760 INFO  (jetty-launcher-71314-thread-3) [n:127.0.0.1:38619_solr     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 8994844 INFO  (jetty-launcher-71314-thread-3) [n:127.0.0.1:38619_solr     ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:38619_solr
   [junit4]   2> 8994845 INFO  (jetty-launcher-71314-thread-3) [n:127.0.0.1:38619_solr     ] o.a.s.c.Overseer Overseer (id=75903699413630987-127.0.0.1:38619_solr-n_0000000000) starting
   [junit4]   2> 8994890 INFO  (jetty-launcher-71314-thread-2) [n:127.0.0.1:45951_solr     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 8994932 INFO  (OverseerStateUpdate-75903699413630987-127.0.0.1:38619_solr-n_0000000000) [n:127.0.0.1:38619_solr     ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:38619_solr
   [junit4]   2> 8994952 INFO  (jetty-launcher-71314-thread-3) [n:127.0.0.1:38619_solr     ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38619_solr
   [junit4]   2> 8994980 INFO  (zkConnectionManagerCallback-71354-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 8994980 INFO  (jetty-launcher-71314-thread-2) [n:127.0.0.1:45951_solr     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 8994992 WARN  (jetty-launcher-71314-thread-3) [n:127.0.0.1:38619_solr     ] o.a.s.c.CoreContainer Not all security plugins configured!  authentication=disabled authorization=disabled.  Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external.  See https://s.apache.org/solrsecurity for more info
   [junit4]   2> 8995089 INFO  (zkCallback-71349-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 8995109 INFO  (jetty-launcher-71314-thread-3) [n:127.0.0.1:38619_solr     ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 8995239 INFO  (jetty-launcher-71314-thread-3) [n:127.0.0.1:38619_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38619.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@79283900
   [junit4]   2> 8995276 INFO  (jetty-launcher-71314-thread-3) [n:127.0.0.1:38619_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38619.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@79283900
   [junit4]   2> 8995276 INFO  (jetty-launcher-71314-thread-3) [n:127.0.0.1:38619_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38619.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@79283900
   [junit4]   2> 8995314 INFO  (jetty-launcher-71314-thread-3) [n:127.0.0.1:38619_solr     ] o.a.s.p.PackageLoader /packages.json updated to version -1
   [junit4]   2> 8995316 INFO  (jetty-launcher-71314-thread-3) [n:127.0.0.1:38619_solr     ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.update.processor.DimensionalRoutedAliasUpdateProcessorTest_94A90A26848C56A1-001/tempDir-001/node3/.
   [junit4]   2> 8995317 INFO  (jetty-launcher-71314-thread-2) [n:127.0.0.1:45951_solr     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 8995353 INFO  (jetty-launcher-71314-thread-2) [n:127.0.0.1:45951_solr     ] o.a.s.c.ZkController Publish node=127.0.0.1:45951_solr as DOWN
   [junit4]   2> 8995366 INFO  (jetty-launcher-71314-thread-2) [n:127.0.0.1:45951_solr     ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 8995366 INFO  (jetty-launcher-71314-thread-2) [n:127.0.0.1:45951_solr     ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45951_solr
   [junit4]   2> 8995368 INFO  (zkCallback-71349-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 8995369 WARN  (jetty-launcher-71314-thread-2) [n:127.0.0.1:45951_solr     ] o.a.s.c.CoreContainer Not all security plugins configured!  authentication=disabled authorization=disabled.  Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external.  See https://s.apache.org/solrsecurity for more info
   [junit4]   2> 8995381 INFO  (zkCallback-71353-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 8995558 INFO  (jetty-launcher-71314-thread-2) [n:127.0.0.1:45951_solr     ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 8995575 INFO  (jetty-launcher-71314-thread-1) [n:127.0.0.1:35222_solr     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 8995610 INFO  (jetty-launcher-71314-thread-1) [n:127.0.0.1:35222_solr     ] o.a.s.c.ZkController Publish node=127.0.0.1:35222_solr as DOWN
   [junit4]   2> 8995611 INFO  (jetty-launcher-71314-thread-1) [n:127.0.0.1:35222_solr     ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 8995612 INFO  (jetty-launcher-71314-thread-1) [n:127.0.0.1:35222_solr     ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35222_solr
   [junit4]   2> 8995612 INFO  (zkCallback-71349-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 8995613 INFO  (zkCallback-71353-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 8995626 WARN  (jetty-launcher-71314-thread-1) [n:127.0.0.1:35222_solr     ] o.a.s.c.CoreContainer Not all security plugins configured!  authentication=disabled authorization=disabled.  Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external.  See https://s.apache.org/solrsecurity for more info
   [junit4]   2> 8995629 INFO  (zkCallback-71340-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 8995683 INFO  (jetty-launcher-71314-thread-2) [n:127.0.0.1:45951_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45951.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@79283900
   [junit4]   2> 8995720 INFO  (jetty-launcher-71314-thread-2) [n:127.0.0.1:45951_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45951.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@79283900
   [junit4]   2> 8995720 INFO  (jetty-launcher-71314-thread-2) [n:127.0.0.1:45951_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45951.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@79283900
   [junit4]   2> 8995722 INFO  (jetty-launcher-71314-thread-2) [n:127.0.0.1:45951_solr     ] o.a.s.p.PackageLoader /packages.json updated to version -1
   [junit4]   2> 8995722 INFO  (jetty-launcher-71314-thread-2) [n:127.0.0.1:45951_solr     ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.update.processor.DimensionalRoutedAliasUpdateProcessorTest_94A90A26848C56A1-001/tempDir-001/node2/.
   [junit4]   2> 8995732 INFO  (jetty-launcher-71314-thread-1) [n:127.0.0.1:35222_solr     ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 8995803 INFO  (jetty-launcher-71314-thread-1) [n:127.0.0.1:35222_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35222.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@79283900
   [junit4]   2> 8995854 INFO  (jetty-launcher-71314-thread-1) [n:127.0.0.1:35222_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35222.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@79283900
   [junit4]   2> 8995854 INFO  (jetty-launcher-71314-thread-1) [n:127.0.0.1:35222_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35222.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@79283900
   [junit4]   2> 8995865 INFO  (jetty-launcher-71314-thread-1) [n:127.0.0.1:35222_solr     ] o.a.s.p.PackageLoader /packages.json updated to version -1
   [junit4]   2> 8995941 INFO  (jetty-launcher-71314-thread-1) [n:127.0.0.1:35222_solr     ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.update.processor.DimensionalRoutedAliasUpdateProcessorTest_94A90A26848C56A1-001/tempDir-001/node1/.
   [junit4]   2> 8996290 INFO  (jetty-launcher-71314-thread-4) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
   [junit4]   2> 8996291 WARN  (jetty-launcher-71314-thread-4) [     ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
   [junit4]   2> 8996292 WARN  (jetty-launcher-71314-thread-4) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@8e98c7[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 8996292 WARN  (jetty-launcher-71314-thread-4) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@8e98c7[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 8996294 WARN  (jetty-launcher-71314-thread-4) [     ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
   [junit4]   2> 8996295 WARN  (jetty-launcher-71314-thread-4) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@31a47a03[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 8996295 WARN  (jetty-launcher-71314-thread-4) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@31a47a03[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 8996308 INFO  (jetty-launcher-71314-thread-4) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45944/solr
   [junit4]   2> 8996309 INFO  (jetty-launcher-71314-thread-4) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 8996310 INFO  (zkConnectionManagerCallback-71368-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 8996310 INFO  (jetty-launcher-71314-thread-4) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 8996412 INFO  (jetty-launcher-71314-thread-4) [n:127.0.0.1:34283_solr     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 8996413 INFO  (zkConnectionManagerCallback-71370-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 8996413 INFO  (jetty-launcher-71314-thread-4) [n:127.0.0.1:34283_solr     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 8996417 INFO  (jetty-launcher-71314-thread-4) [n:127.0.0.1:34283_solr     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 8996420 INFO  (jetty-launcher-71314-thread-4) [n:127.0.0.1:34283_solr     ] o.a.s.c.ZkController Publish node=127.0.0.1:34283_solr as DOWN
   [junit4]   2> 8996421 INFO  (jetty-launcher-71314-thread-4) [n:127.0.0.1:34283_solr     ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 8996421 INFO  (jetty-launcher-71314-thread-4) [n:127.0.0.1:34283_solr     ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:34283_solr
   [junit4]   2> 8996422 INFO  (zkCallback-71353-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 8996422 INFO  (zkCallback-71340-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 8996422 INFO  (zkCallback-71349-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 8996423 INFO  (zkCallback-71369-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 8996423 WARN  (jetty-launcher-71314-thread-4) [n:127.0.0.1:34283_solr     ] o.a.s.c.CoreContainer Not all security plugins configured!  authentication=disabled authorization=disabled.  Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external.  See https://s.apache.org/solrsecurity for more info
   [junit4]   2> 8996463 INFO  (jetty-launcher-71314-thread-4) [n:127.0.0.1:34283_solr     ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 8996502 INFO  (jetty-launcher-71314-thread-4) [n:127.0.0.1:34283_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_34283.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@79283900
   [junit4]   2> 8996518 INFO  (jetty-launcher-71314-thread-4) [n:127.0.0.1:34283_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_34283.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@79283900
   [junit4]   2> 8996518 INFO  (jetty-launcher-71314-thread-4) [n:127.0.0.1:34283_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_34283.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@79283900
   [junit4]   2> 8996532 INFO  (jetty-launcher-71314-thread-4) [n:127.0.0.1:34283_solr     ] o.a.s.p.PackageLoader /packages.json updated to version -1
   [junit4]   2> 8996533 INFO  (jetty-launcher-71314-thread-4) [n:127.0.0.1:34283_solr     ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.update.processor.DimensionalRoutedAliasUpdateProcessorTest_94A90A26848C56A1-001/tempDir-001/node4/.
   [junit4]   2> 8996646 INFO  (TEST-DimensionalRoutedAliasUpdateProcessorTest.testCatTime-seed#[94A90A26848C56A1]) [     ] o.a.s.c.MiniSolrCloudCluster waitForAllNodes: numServers=4
   [junit4]   2> 8996647 INFO  (TEST-DimensionalRoutedAliasUpdateProcessorTest.testCatTime-seed#[94A90A26848C56A1]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 8996648 INFO  (zkConnectionManagerCallback-71379-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 8996648 INFO  (TEST-DimensionalRoutedAliasUpdateProcessorTest.testCatTime-seed#[94A90A26848C56A1]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 8996649 INFO  (TEST-DimensionalRoutedAliasUpdateProcessorTest.testCatTime-seed#[94A90A26848C56A1]) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 8996650 INFO  (TEST-DimensionalRoutedAliasUpdateProcessorTest.testCatTime-seed#[94A90A26848C56A1]) [     ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45944/solr ready
   [junit4]   2> 8996651 INFO  (TEST-DimensionalRoutedAliasUpdateProcessorTest.testCatTime-seed#[94A90A26848C56A1]) [     ] o.a.s.u.p.DimensionalRoutedAliasUpdateProcessorTest SolrClient: org.apache.solr.client.solrj.impl.CloudSolrClient@543c9a15
   [junit4]   2> 8996651 INFO  (TEST-DimensionalRoutedAliasUpdateProcessorTest.testCatTime-seed#[94A90A26848C56A1]) [     ] o.a.s.u.p.DimensionalRoutedAliasUpdateProcessorTest ClusterStateProvider 127.0.0.1:45944/solr
   [junit4]   2> 8996652 INFO  (TEST-DimensionalRoutedAliasUpdateProcessorTest.testCatTime-seed#[94A90A26848C56A1]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 8996653 INFO  (zkConnectionManagerCallback-71384-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 8996653 INFO  (TEST-DimensionalRoutedAliasUpdateProcessorTest.testCatTime-seed#[94A90A26848C56A1]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 8996666 INFO  (TEST-DimensionalRoutedAliasUpdateProcessorTest.testCatTime-seed#[94A90A26848C56A1]) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 8996667 INFO  (TEST-DimensionalRoutedAliasUpdateProcessorTest.testCatTime-seed#[94A90A26848C56A1]) [     ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45944/solr ready
   [junit4]   2> 8996732 INFO  (qtp1149499270-160729) [n:127.0.0.1:45951_solr     ] o.a.s.h.a.ConfigSetsHandler Invoked ConfigSet Action :create with params baseConfigSet=_default&name=testCatTime&action=CREATE&wt=javabin&version=2 
   [junit4]   2> 8996735 INFO  (OverseerThreadFactory-19569-thread-1-processing-n:127.0.0.1:38619_solr) [n:127.0.0.1:38619_solr     ] o.a.s.c.OverseerConfigSetMessageHandler OverseerConfigSetMessageHandler.processMessage : create , {
   [junit4]   2>   "name":"testCatTime",
   [junit4]   2>   "baseConfigSet":"_default",
   [junit4]   2>   "immutable":"false",
   [junit4]   2>   "operation":"configsets:create"}
   [junit4]   2> 8996736 INFO  (OverseerThreadFactory-19569-thread-1-processing-n:127.0.0.1:38619_solr) [n:127.0.0.1:38619_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/protwords.txt to /configs/testCatTime/protwords.txt
   [junit4]   2> 8996738 INFO  (OverseerThreadFactory-19569-thread-1-processing-n:127.0.0.1:38619_solr) [n:127.0.0.1:38619_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/managed-schema to /configs/testCatTime/managed-schema
   [junit4]   2> 8996748 INFO  (OverseerThreadFactory-19569-thread-1-processing-n:127.0.0.1:38619_solr) [n:127.0.0.1:38619_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/solrconfig.xml to /configs/testCatTime/solrconfig.xml
   [junit4]   2> 8996750 INFO  (OverseerThreadFactory-19569-thread-1-processing-n:127.0.0.1:38619_solr) [n:127.0.0.1:38619_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/contractions_it.txt to /configs/testCatTime/lang/contractions_it.txt
   [junit4]   2> 8996763 INFO  (OverseerThreadFactory-19569-thread-1-processing-n:127.0.0.1:38619_solr) [n:127.0.0.1:38619_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_no.txt to /configs/testCatTime/lang/stopwords_no.txt
   [junit4]   2> 8996765 INFO  (OverseerThreadFactory-19569-thread-1-processing-n:127.0.0.1:38619_solr) [n:127.0.0.1:38619_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_hy.txt to /configs/testCatTime/lang/stopwords_hy.txt
   [junit4]   2> 8996767 INFO  (OverseerThreadFactory-19569-thread-1-processing-n:127.0.0.1:38619_solr) [n:127.0.0.1:38619_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stemdict_nl.txt to /configs/testCatTime/lang/stemdict_nl.txt
   [junit4]   2> 8996768 INFO  (OverseerThreadFactory-19569-thread-1-processing-n:127.0.0.1:38619_solr) [n:127.0.0.1:38619_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/contractions_ca.txt to /configs/testCatTime/lang/contractions_ca.txt
   [junit4]   2> 8996770 INFO  (OverseerThreadFactory-19569-thread-1-processing-n:127.0.0.1:38619_solr) [n:127.0.0.1:38619_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_id.txt to /configs/testCatTime/lang/stopwords_id.txt
   [junit4]   2> 8996784 INFO  (OverseerThreadFactory-19569-thread-1-processing-n:127.0.0.1:38619_solr) [n:127.0.0.1:38619_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_et.txt to /configs/testCatTime/lang/stopwords_et.txt
   [junit4]   2> 8996785 INFO  (OverseerThreadFactory-19569-thread-1-processing-n:127.0.0.1:38619_solr) [n:127.0.0.1:38619_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_ga.txt to /configs/testCatTime/lang/stopwords_ga.txt
   [junit4]   2> 8996786 INFO  (OverseerThreadFactory-19569-thread-1-processing-n:127.0.0.1:38619_solr) [n:127.0.0.1:38619_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_da.txt to /configs/testCatTime/lang/stopwords_da.txt
   [junit4]   2> 8996788 INFO  (OverseerThreadFactory-19569-thread-1-processing-n:127.0.0.1:38619_solr) [n:127.0.0.1:38619_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_hi.txt to /configs/testCatTime/lang/stopwords_hi.txt
   [junit4]   2> 8996789 INFO  (OverseerThreadFactory-19569-thread-1-processing-n:127.0.0.1:38619_solr) [n:127.0.0.1:38619_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_ja.txt to /configs/testCatTime/lang/stopwords_ja.txt
   [junit4]   2> 8996791 INFO  (OverseerThreadFactory-19569-thread-1-processing-n:127.0.0.1:38619_solr) [n:127.0.0.1:38619_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_pt.txt to /configs/testCatTime/lang/stopwords_pt.txt
   [junit4]   2> 8996801 INFO  (OverseerThreadFactory-19569-thread-1-processing-n:127.0.0.1:38619_solr) [n:127.0.0.1:38619_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_hu.txt to /configs/testCatTime/lang/stopwords_hu.txt
   [junit4]   2> 8996802 INFO  (OverseerThreadFactory-19569-thread-1-processing-n:127.0.0.1:38619_solr) [n:127.0.0.1:38619_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_el.txt to /configs/testCatTime/lang/stopwords_el.txt
   [junit4]   2> 8996804 INFO  (OverseerThreadFactory-19569-thread-1-processing-n:127.0.0.1:38619_solr) [n:127.0.0.1:38619_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_ru.txt to /configs/testCatTime/lang/stopwords_ru.txt
   [junit4]   2> 8996805 INFO  (OverseerThreadFactory-19569-thread-1-processing-n:127.0.0.1:38619_solr) [n:127.0.0.1:38619_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_tr.txt to /configs/testCatTime/lang/stopwords_tr.txt
   [junit4]   2> 8996807 INFO  (OverseerThreadFactory-19569-thread-1-processing-n:127.0.0.1:38619_solr) [n:127.0.0.1:38619_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_ar.txt to /configs/testCatTime/lang/stopwords_ar.txt
   [junit4]   2> 8996817 INFO  (OverseerThreadFactory-19569-thread-1-processing-n:127.0.0.1:38619_solr) [n:127.0.0.1:38619_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/userdict_ja.txt to /configs/testCatTime/lang/userdict_ja.txt
   [junit4]   2> 8996818 INFO  (OverseerThreadFactory-19569-thread-1-processing-n:127.0.0.1:38619_solr) [n:127.0.0.1:38619_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_it.txt to /configs/testCatTime/lang/stopwords_it.txt
   [junit4]   2> 8996820 INFO  (OverseerThreadFactory-19569-thread-1-processing-n:127.0.0.1:38619_solr) [n:127.0.0.1:38619_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_eu.txt to /configs/testCatTime/lang/stopwords_eu.txt
   [junit4]   2> 8996821 INFO  (OverseerThreadFactory-19569-thread-1-processing-n:127.0.0.1:38619_solr) [n:127.0.0.1:38619_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_cz.txt to /configs/testCatTime/lang/stopwords_cz.txt
   [junit4]   2> 8996823 INFO  (OverseerThreadFactory-19569-thread-1-processing-n:127.0.0.1:38619_solr) [n:127.0.0.1:38619_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stoptags_ja.txt to /configs/testCatTime/lang/stoptags_ja.txt
   [junit4]   2> 8996834 INFO  (OverseerThreadFactory-19569-thread-1-processing-n:127.0.0.1:38619_solr) [n:127.0.0.1:38619_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/contractions_fr.txt to /configs/testCatTime/lang/contractions_fr.txt
   [junit4]   2> 8996835 INFO  (OverseerThreadFactory-19569-thread-1-processing-n:127.0.0.1:38619_solr) [n:127.0.0.1:38619_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_fa.txt to /configs/testCatTime/lang/stopwords_fa.txt
   [junit4]   2> 8996845 INFO  (OverseerThreadFactory-19569-thread-1-processing-n:127.0.0.1:38619_solr) [n:127.0.0.1:38619_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_de.txt to /configs/testCatTime/lang/stopwords_de.txt
   [junit4]   2> 8996846 INFO  (OverseerThreadFactory-19569-thread-1-processing-n:127.0.0.1:38619_solr) [n:127.0.0.1:38619_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/contractions_ga.txt to /configs/testCatTime/lang/contractions_ga.txt
   [junit4]   2> 8996847 INFO  (OverseerThreadFactory-19569-thread-1-processing-n:127.0.0.1:38619_solr) [n:127.0.0.1:38619_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_ca.txt to /configs/testCatTime/lang/stopwords_ca.txt
   [junit4]   2> 8996849 INFO  (OverseerThreadFactory-19569-thread-1-processing-n:127.0.0.1:38619_solr) [n:127.0.0.1:38619_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_nl.txt to /configs/testCatTime/lang/stopwords_nl.txt
   [junit4]   2> 8996850 INFO  (OverseerThreadFactory-19569-thread-1-processing-n:127.0.0.1:38619_solr) [n:127.0.0.1:38619_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_bg.txt to /configs/testCatTime/lang/stopwords_bg.txt
   [junit4]   2> 8996852 INFO  (OverseerThreadFactory-19569-thread-1-processing-n:127.0.0.1:38619_solr) [n:127.0.0.1:38619_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_es.txt to /configs/testCatTime/lang/stopwords_es.txt
   [junit4]   2> 8996853 INFO  (OverseerThreadFactory-19569-thread-1-processing-n:127.0.0.1:38619_solr) [n:127.0.0.1:38619_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_en.txt to /configs/testCatTime/lang/stopwords_en.txt
   [junit4]   2> 8996855 INFO  (OverseerThreadFactory-19569-thread-1-processing-n:127.0.0.1:38619_solr) [n:127.0.0.1:38619_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_lv.txt to /configs/testCatTime/lang/stopwords_lv.txt
   [junit4]   2> 8996870 INFO  (OverseerThreadFactory-19569-thread-1-processing-n:127.0.0.1:38619_solr) [n:127.0.0.1:38619_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_th.txt to /configs/testCatTime/lang/stopwords_th.txt
   [junit4]   2> 8996871 INFO  (OverseerThreadFactory-19569-thread-1-processing-n:127.0.0.1:38619_solr) [n:127.0.0.1:38619_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_ro.txt to /configs/testCatTime/lang/stopwords_ro.txt
   [junit4]   2> 8996873 INFO  (OverseerThreadFactory-19569-thread-1-processing-n:127.0.0.1:38619_solr) [n:127.0.0.1:38619_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_fr.txt to /configs/testCatTime/lang/stopwords_fr.txt
   [junit4]   2> 8996874 INFO  (OverseerThreadFactory-19569-thread-1-processing-n:127.0.0.1:38619_solr) [n:127.0.0.1:38619_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_sv.txt to /configs/testCatTime/lang/stopwords_sv.txt
   [junit4]   2> 8996875 INFO  (OverseerThreadFactory-19569-thread-1-processing-n:127.0.0.1:38619_solr) [n:127.0.0.1:38619_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_fi.txt to /configs/testCatTime/lang/stopwords_fi.txt
   [junit4]   2> 8996885 INFO  (OverseerThreadFactory-19569-thread-1-processing-n:127.0.0.1:38619_solr) [n:127.0.0.1:38619_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/hyphenations_ga.txt to /configs/testCatTime/lang/hyphenations_ga.txt
   [junit4]   2> 8996887 INFO  (OverseerThreadFactory-19569-thread-1-processing-n:127.0.0.1:38619_solr) [n:127.0.0.1:38619_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_gl.txt to /configs/testCatTime/lang/stopwords_gl.txt
   [junit4]   2> 8996888 INFO  (OverseerThreadFactory-19569-thread-1-processing-n:127.0.0.1:38619_solr) [n:127.0.0.1:38619_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/synonyms.txt to /configs/testCatTime/synonyms.txt
   [junit4]   2> 8996889 INFO  (OverseerThreadFactory-19569-thread-1-processing-n:127.0.0.1:38619_solr) [n:127.0.0.1:38619_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/stopwords.txt to /configs/testCatTime/stopwords.txt
   [junit4]   2> 8996891 INFO  (qtp1149499270-160729) [n:127.0.0.1:45951_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/configs params={baseConfigSet=_default&name=testCatTime&action=CREATE&wt=javabin&version=2} status=0 QTime=159
   [junit4]   2> 8996893 INFO  (qtp1149499270-160722) [n:127.0.0.1:45951_solr     ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=testCatTime&name=testCatTime&nrtReplicas=1&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 8996908 INFO  (OverseerThreadFactory-19569-thread-2-processing-n:127.0.0.1:38619_solr) [n:127.0.0.1:38619_solr     ] o.a.s.c.a.c.CreateCollectionCmd Create collection testCatTime
   [junit4]   2> 8996908 INFO  (OverseerCollectionConfigSetProcessor-75903699413630987-127.0.0.1:38619_solr-n_0000000000) [n:127.0.0.1:38619_solr     ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 8997040 INFO  (OverseerStateUpdate-75903699413630987-127.0.0.1:38619_solr-n_0000000000) [n:127.0.0.1:38619_solr     ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"testCatTime",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"testCatTime_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:38619/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 8997321 INFO  (qtp628470363-160738) [n:127.0.0.1:38619_solr    x:testCatTime_shard1_replica_n1 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node2&collection.configName=testCatTime&newCollection=true&name=testCatTime_shard1_replica_n1&action=CREATE&numShards=1&collection=testCatTime&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 8997321 INFO  (qtp628470363-160738) [n:127.0.0.1:38619_solr    x:testCatTime_shard1_replica_n1 ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 8998426 INFO  (qtp628470363-160738) [n:127.0.0.1:38619_solr c:testCatTime s:shard1 r:core_node2 x:testCatTime_shard1_replica_n1 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.5.0
   [junit4]   2> 8998495 INFO  (qtp628470363-160738) [n:127.0.0.1:38619_solr c:testCatTime s:shard1 r:core_node2 x:testCatTime_shard1_replica_n1 ] o.a.s.s.IndexSchema [testCatTime_shard1_replica_n1] Schema name=default-config
   [junit4]   2> 8999013 INFO  (qtp628470363-160738) [n:127.0.0.1:38619_solr c:testCatTime s:shard1 r:core_node2 x:testCatTime_shard1_replica_n1 ] o.a.s.s.IndexSchema Loaded schema default-config/1.6 with uniqueid field id
   [junit4]   2> 8999104 INFO  (qtp628470363-160738) [n:127.0.0.1:38619_solr c:testCatTime s:shard1 r:core_node2 x:testCatTime_shard1_replica_n1 ] o.a.s.c.CoreContainer Creating SolrCore 'testCatTime_shard1_replica_n1' using configuration from collection testCatTime, trusted=true
   [junit4]   2> 8999104 INFO  (qtp628470363-160738) [n:127.0.0.1:38619_solr c:testCatTime s:shard1 r:core_node2 x:testCatTime_shard1_replica_n1 ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38619.solr.core.testCatTime.shard1.replica_n1' (registry 'solr.core.testCatTime.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@79283900
   [junit4]   2> 8999104 INFO  (qtp628470363-160738) [n:127.0.0.1:38619_solr c:testCatTime s:shard1 r:core_node2 x:testCatTime_shard1_replica_n1 ] o.a.s.c.SolrCore [[testCatTime_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.update.processor.DimensionalRoutedAliasUpdateProcessorTest_94A90A26848C56A1-001/tempDir-001/node3/testCatTime_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.update.processor.DimensionalRoutedAliasUpdateProcessorTest_94A90A26848C56A1-001/tempDir-001/node3/./testCatTime_shard1_replica_n1/data/]
   [junit4]   2> 8999296 INFO  (qtp628470363-160738) [n:127.0.0.1:38619_solr c:testCatTime s:shard1 r:core_node2 x:testCatTime_shard1_replica_n1 ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 8999296 INFO  (qtp628470363-160738) [n:127.0.0.1:38619_solr c:testCatTime s:shard1 r:core_node2 x:testCatTime_shard1_replica_n1 ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 8999298 INFO  (qtp628470363-160738) [n:127.0.0.1:38619_solr c:testCatTime s:shard1 r:core_node2 x:testCatTime_shard1_replica_n1 ] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms; 
   [junit4]   2> 8999298 INFO  (qtp628470363-160738) [n:127.0.0.1:38619_solr c:testCatTime s:shard1 r:core_node2 x:testCatTime_shard1_replica_n1 ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 8999311 INFO  (qtp628470363-160738) [n:127.0.0.1:38619_solr c:testCatTime s:shard1 r:core_node2 x:testCatTime_shard1_replica_n1 ] o.a.s.s.SolrIndexSearcher Opening [Searcher@3e0f0cb3[testCatTime_shard1_replica_n1] main]
   [junit4]   2> 8999313 INFO  (qtp628470363-160738) [n:127.0.0.1:38619_solr c:testCatTime s:shard1 r:core_node2 x:testCatTime_shard1_replica_n1 ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/testCatTime
   [junit4]   2> 8999313 INFO  (qtp628470363-160738) [n:127.0.0.1:38619_solr c:testCatTime s:shard1 r:core_node2 x:testCatTime_shard1_replica_n1 ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/testCatTime
   [junit4]   2> 8999313 INFO  (qtp628470363-160738) [n:127.0.0.1:38619_solr c:testCatTime s:shard1 r:core_node2 x:testCatTime_shard1_replica_n1 ] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/testCatTime/managed-schema
   [junit4]   2> 8999314 INFO  (qtp628470363-160738) [n:127.0.0.1:38619_solr c:testCatTime s:shard1 r:core_node2 x:testCatTime_shard1_replica_n1 ] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
   [junit4]   2> 8999314 INFO  (qtp628470363-160738) [n:127.0.0.1:38619_solr c:testCatTime s:shard1 r:core_node2 x:testCatTime_shard1_replica_n1 ] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
   [junit4]   2> 8999314 INFO  (qtp628470363-160738) [n:127.0.0.1:38619_solr c:testCatTime s:shard1 r:core_node2 x:testCatTime_shard1_replica_n1 ] o.a.s.s.DirectSolrSpellChecker init: {name=default,field=_text_,classname=solr.DirectSolrSpellChecker,distanceMeasure=internal,accuracy=0.5,maxEdits=2,minPrefix=1,maxInspections=5,minQueryLength=4,maxQueryFrequency=0.01}
   [junit4]   2> 8999328 INFO  (qtp628470363-160738) [n:127.0.0.1:38619_solr c:testCatTime s:shard1 r:core_node2 x:testCatTime_shard1_replica_n1 ] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 8999328 INFO  (qtp628470363-160738) [n:127.0.0.1:38619_solr c:testCatTime s:shard1 r:core_node2 x:testCatTime_shard1_replica_n1 ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1652922269811343360
   [junit4]   2> 8999330 INFO  (searcherExecutor-19589-thread-1-processing-n:127.0.0.1:38619_solr x:testCatTime_shard1_replica_n1 c:testCatTime s:shard1 r:core_node2) [n:127.0.0.1:38619_solr c:testCatTime s:shard1 r:core_node2 x:testCatTime_shard1_replica_n1 ] o.a.s.c.QuerySenderListener QuerySenderListener sending requests to Searcher@3e0f0cb3[testCatTime_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 8999330 INFO  (searcherExecutor-19589-thread-1-processing-n:127.0.0.1:38619_solr x:testCatTime_shard1_replica_n1 c:testCatTime s:shard1 r:core_node2) [n:127.0.0.1:38619_solr c:testCatTime s:shard1 r:core_node2 x:testCatTime_shard1_replica_n1 ] o.a.s.c.QuerySenderListener QuerySenderListener done.
   [junit4]   2> 8999330 INFO  (searcherExecutor-19589-thread-1-processing-n:127.0.0.1:38619_solr x:testCatTime_shard1_replica_n1 c:testCatTime s:shard1 r:core_node2) [n:127.0.0.1:38619_solr c:testCatTime s:shard1 r:core_node2 x:testCatTime_shard1_replica_n1 ] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
   [junit4]   2> 8999331 INFO  (searcherExecutor-19589-thread-1-processing-n:127.0.0.1:38619_solr x:testCatTime_shard1_replica_n1 c:testCatTime s:shard1 r:core_node2) [n:127.0.0.1:38619_solr c:testCatTime s:shard1 r:core_node2 x:testCatTime_shard1_replica_n1 ] o.a.s.c.SolrCore [testCatTime_shard1_replica_n1] Registered new searcher Searcher@3e0f0cb3[testCatTime_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 8999346 INFO  (qtp628470363-160738) [n:127.0.0.1:38619_solr c:testCatTime s:shard1 r:core_node2 x:testCatTime_shard1_replica_n1 ] o.a.s.c.ZkShardTerms Successful update of terms at /collections/testCatTime/terms/shard1 to Terms{values={core_node2=0}, version=0}
   [junit4]   2> 8999346 INFO  (qtp628470363-160738) [n:127.0.0.1:38619_solr c:testCatTime s:shard1 r:core_node2 x:testCatTime_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/testCatTime/leaders/shard1
   [junit4]   2> 8999442 INFO  (qtp628470363-160738) [n:127.0.0.1:38619_solr c:testCatTime s:shard1 r:core_node2 x:testCatTime_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 8999442 INFO  (qtp628470363-160738) [n:127.0.0.1:38619_solr c:testCatTime s:shard1 r:core_node2 x:testCatTime_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 8999442 INFO  (qtp628470363-160738) [n:127.0.0.1:38619_solr c:testCatTime s:shard1 r:core_node2 x:testCatTime_shard1_replica_n1 ] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:38619/solr/testCatTime_shard1_replica_n1/
   [junit4]   2> 8999442 INFO  (qtp628470363-160738) [n:127.0.0.1:38619_solr c:testCatTime s:shard1 r:core_node2 x:testCatTime_shard1_replica_n1 ] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 8999442 INFO  (qtp628470363-160738) [n:127.0.0.1:38619_solr c:testCatTime s:shard1 r:core_node2 x:testCatTime_shard1_replica_n1 ] o.a.s.c.SyncStrategy https://127.0.0.1:38619/solr/testCatTime_shard1_replica_n1/ has no replicas
   [junit4]   2> 8999442 INFO  (qtp628470363-160738) [n:127.0.0.1:38619_solr c:testCatTime s:shard1 r:core_node2 x:testCatTime_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/testCatTime/leaders/shard1/leader after winning as /collections/testCatTime/leader_elect/shard1/election/75903699413630987-core_node2-n_0000000000
   [junit4]   2> 8999444 INFO  (qtp628470363-160738) [n:127.0.0.1:38619_solr c:testCatTime s:shard1 r:core_node2 x:testCatTime_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:38619/solr/testCatTime_shard1_replica_n1/ shard1
   [junit4]   2> 8999546 INFO  (zkCallback-71349-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCatTime/stat

[...truncated too long message...]

 1> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   1> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
   [junit4]   1> 	at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
   [junit4]   1> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:717)
   [junit4]   1> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
   [junit4]   1> 	at org.eclipse.jetty.server.Server.handle(Server.java:500)
   [junit4]   1> 	at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)
   [junit4]   1> 	at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547)
   [junit4]   1> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)
   [junit4]   1> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:270)
   [junit4]   1> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311)
   [junit4]   1> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
   [junit4]   1> 	at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:117)
   [junit4]   1> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
   [junit4]   1> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
   [junit4]   1> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   1> 
   [junit4]   1> 19:15:09.485 [qtp466669100-102] ERROR org.apache.solr.servlet.HttpSolrCall - null:java.io.IOException: Task queue processing has stalled for 20057 ms with 0 remaining elements to process.
   [junit4]   1> 	at org.apache.solr.client.solrj.impl.ConcurrentUpdateHttp2SolrClient.blockUntilFinished(ConcurrentUpdateHttp2SolrClient.java:501)
   [junit4]   1> 	at org.apache.solr.update.StreamingSolrClients.blockUntilFinished(StreamingSolrClients.java:87)
   [junit4]   1> 	at org.apache.solr.update.SolrCmdDistributor.blockAndDoRetries(SolrCmdDistributor.java:265)
   [junit4]   1> 	at org.apache.solr.update.SolrCmdDistributor.distribCommit(SolrCmdDistributor.java:251)
   [junit4]   1> 	at org.apache.solr.update.processor.DistributedZkUpdateProcessor.processCommit(DistributedZkUpdateProcessor.java:201)
   [junit4]   1> 	at org.apache.solr.handler.RequestHandlerUtils.handleCommit(RequestHandlerUtils.java:69)
   [junit4]   1> 	at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:72)
   [junit4]   1> 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:211)
   [junit4]   1> 	at org.apache.solr.core.SolrCore.execute(SolrCore.java:2596)
   [junit4]   1> 	at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:799)
   [junit4]   1> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:578)
   [junit4]   1> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:419)
   [junit4]   1> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:351)
   [junit4]   1> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)
   [junit4]   1> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166)
   [junit4]   1> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)
   [junit4]   1> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545)
   [junit4]   1> 	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
   [junit4]   1> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1607)
   [junit4]   1> 	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
   [junit4]   1> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1297)
   [junit4]   1> 	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
   [junit4]   1> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485)
   [junit4]   1> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1577)
   [junit4]   1> 	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
   [junit4]   1> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1212)
   [junit4]   1> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   1> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
   [junit4]   1> 	at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
   [junit4]   1> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:717)
   [junit4]   1> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
   [junit4]   1> 	at org.eclipse.jetty.server.Server.handle(Server.java:500)
   [junit4]   1> 	at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)
   [junit4]   1> 	at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547)
   [junit4]   1> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)
   [junit4]   1> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:270)
   [junit4]   1> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311)
   [junit4]   1> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
   [junit4]   1> 	at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:117)
   [junit4]   1> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
   [junit4]   1> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
   [junit4]   1> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   1> 
   [junit4]   1> 19:15:09.517 [SUITE-RangeFacetTest-seed#[81932C87837F5555]-worker] ERROR org.apache.solr.client.solrj.impl.BaseCloudSolrClient - Request to collection [collection1] failed due to (500) org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:41575/solr/collection1: Task queue processing has stalled for 20057 ms with 0 remaining elements to process., retry=0 commError=false errorCode=500 
   [junit4]   2> Dec 14, 2019 7:15:10 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Lucene84, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@603d88e1), locale=ar-LY, timezone=Asia/Shanghai
   [junit4]   2> NOTE: Linux 4.4.0-112-generic amd64/Oracle Corporation 1.8.0_191 (64-bit)/cpus=4,threads=1,free=137945104,total=256376832
   [junit4]   2> NOTE: All tests run in this JVM: [RoundFunctionTest, ConstantValueTest, CastingDateValueStreamTest, StringFieldsTest, RangeFacetTest]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RangeFacetTest -Dtests.seed=81932C87837F5555 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/test-data/enwiki.random.lines.txt -Dtests.locale=ar-LY -Dtests.timezone=Asia/Shanghai -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   0.00s J2 | RangeFacetTest (suite) <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:41575/solr/collection1: Task queue processing has stalled for 20057 ms with 0 remaining elements to process.
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([81932C87837F5555]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:665)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:265)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:368)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:296)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.sendRequest(BaseCloudSolrClient.java:1143)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.requestWithRetryOnStaleState(BaseCloudSolrClient.java:906)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.request(BaseCloudSolrClient.java:838)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
   [junit4]    > 	at org.apache.solr.client.solrj.request.UpdateRequest.commit(UpdateRequest.java:240)
   [junit4]    > 	at org.apache.solr.analytics.SolrAnalyticsTestCase.setupCollection(SolrAnalyticsTestCase.java:68)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4] Completed [18/78 (1!)] on J2 in 59.68s, 0 tests, 1 error <<< FAILURES!

[...truncated 6314 lines...]
BUILD FAILED
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/build.xml:652: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/build.xml:586: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid18362.hprof
* java_pid18367.hprof

Total time: 539 minutes 39 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any

[JENKINS] Lucene-Solr-NightlyTests-8.x - Build # 295 - Still Unstable

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-8.x/295/

3 tests failed.
FAILED:  org.apache.solr.cloud.api.collections.ShardSplitTest.testSplitWithChaosMonkey

Error Message:
Address already in use

Stack Trace:
java.net.BindException: Address already in use
	at __randomizedtesting.SeedInfo.seed([32A32D8993FC5BEE:B984FE58D2FAF06A]:0)
	at sun.nio.ch.Net.bind0(Native Method)
	at sun.nio.ch.Net.bind(Net.java:433)
	at sun.nio.ch.Net.bind(Net.java:425)
	at sun.nio.ch.ServerSocketChannelImpl.bind(ServerSocketChannelImpl.java:223)
	at sun.nio.ch.ServerSocketAdaptor.bind(ServerSocketAdaptor.java:74)
	at org.eclipse.jetty.server.ServerConnector.openAcceptChannel(ServerConnector.java:342)
	at org.eclipse.jetty.server.ServerConnector.open(ServerConnector.java:308)
	at org.eclipse.jetty.server.AbstractNetworkConnector.doStart(AbstractNetworkConnector.java:80)
	at org.eclipse.jetty.server.ServerConnector.doStart(ServerConnector.java:236)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
	at org.eclipse.jetty.server.Server.doStart(Server.java:396)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.retryOnPortBindFailure(JettySolrRunner.java:567)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:506)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:474)
	at org.apache.solr.cloud.api.collections.ShardSplitTest.testSplitWithChaosMonkey(ShardSplitTest.java:499)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1082)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1054)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.api.collections.ShardSplitTest

Error Message:
10 threads leaked from SUITE scope at org.apache.solr.cloud.api.collections.ShardSplitTest:     1) Thread[id=293261, name=qtp1683749906-293261, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=293258, name=qtp1683749906-293258-acceptor-0@4138f962-ServerConnector@60646441{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:45173}, state=RUNNABLE, group=TGRP-ShardSplitTest]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=293342, name=Connector-Scheduler-60646441, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=293263, name=qtp1683749906-293263, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=293257, name=qtp1683749906-293257, state=RUNNABLE, group=TGRP-ShardSplitTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$59/1646479797.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=293262, name=qtp1683749906-293262, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    7) Thread[id=293265, name=Session-HouseKeeper-68180a62, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    8) Thread[id=293260, name=qtp1683749906-293260, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    9) Thread[id=293264, name=qtp1683749906-293264, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)   10) Thread[id=293259, name=qtp1683749906-293259, state=RUNNABLE, group=TGRP-ShardSplitTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$59/1646479797.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 10 threads leaked from SUITE scope at org.apache.solr.cloud.api.collections.ShardSplitTest: 
   1) Thread[id=293261, name=qtp1683749906-293261, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=293258, name=qtp1683749906-293258-acceptor-0@4138f962-ServerConnector@60646441{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:45173}, state=RUNNABLE, group=TGRP-ShardSplitTest]
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
        at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385)
        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=293342, name=Connector-Scheduler-60646441, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=293263, name=qtp1683749906-293263, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=293257, name=qtp1683749906-293257, state=RUNNABLE, group=TGRP-ShardSplitTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$59/1646479797.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=293262, name=qtp1683749906-293262, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   7) Thread[id=293265, name=Session-HouseKeeper-68180a62, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   8) Thread[id=293260, name=qtp1683749906-293260, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   9) Thread[id=293264, name=qtp1683749906-293264, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
  10) Thread[id=293259, name=qtp1683749906-293259, state=RUNNABLE, group=TGRP-ShardSplitTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$59/1646479797.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([32A32D8993FC5BEE]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.api.collections.ShardSplitTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=293260, name=qtp1683749906-293260, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=293261, name=qtp1683749906-293261, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=293258, name=qtp1683749906-293258, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=293264, name=qtp1683749906-293264, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=293342, name=Connector-Scheduler-60646441, state=WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=293263, name=qtp1683749906-293263, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    7) Thread[id=293259, name=qtp1683749906-293259, state=RUNNABLE, group=TGRP-ShardSplitTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$59/1646479797.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)         at java.lang.Thread.run(Thread.java:748)    8) Thread[id=293257, name=qtp1683749906-293257, state=RUNNABLE, group=TGRP-ShardSplitTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$59/1646479797.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)         at java.lang.Thread.run(Thread.java:748)    9) Thread[id=293262, name=qtp1683749906-293262, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)   10) Thread[id=293265, name=Session-HouseKeeper-68180a62, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=293260, name=qtp1683749906-293260, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=293261, name=qtp1683749906-293261, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=293258, name=qtp1683749906-293258, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=293264, name=qtp1683749906-293264, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=293342, name=Connector-Scheduler-60646441, state=WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=293263, name=qtp1683749906-293263, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   7) Thread[id=293259, name=qtp1683749906-293259, state=RUNNABLE, group=TGRP-ShardSplitTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$59/1646479797.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
        at java.lang.Thread.run(Thread.java:748)
   8) Thread[id=293257, name=qtp1683749906-293257, state=RUNNABLE, group=TGRP-ShardSplitTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$59/1646479797.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
        at java.lang.Thread.run(Thread.java:748)
   9) Thread[id=293262, name=qtp1683749906-293262, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
  10) Thread[id=293265, name=Session-HouseKeeper-68180a62, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([32A32D8993FC5BEE]:0)




Build Log:
[...truncated 15731 lines...]
   [junit4] Suite: org.apache.solr.cloud.api.collections.ShardSplitTest
   [junit4]   2> 3391693 INFO  (SUITE-ShardSplitTest-seed#[32A32D8993FC5BEE]-worker) [     ] o.a.s.SolrTestCaseJ4 Created dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.api.collections.ShardSplitTest_32A32D8993FC5BEE-001/data-dir-158-001
   [junit4]   2> 3391693 WARN  (SUITE-ShardSplitTest-seed#[32A32D8993FC5BEE]-worker) [     ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=56 numCloses=56
   [junit4]   2> 3391693 INFO  (SUITE-ShardSplitTest-seed#[32A32D8993FC5BEE]-worker) [     ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 3391695 INFO  (SUITE-ShardSplitTest-seed#[32A32D8993FC5BEE]-worker) [     ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
   [junit4]   2> 3391695 INFO  (SUITE-ShardSplitTest-seed#[32A32D8993FC5BEE]-worker) [     ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 3391695 INFO  (SUITE-ShardSplitTest-seed#[32A32D8993FC5BEE]-worker) [     ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
   [junit4]   2> 3391699 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 3391699 INFO  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 3391699 INFO  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 3391799 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.a.s.c.ZkTestServer start zk server on port:36087
   [junit4]   2> 3391799 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.a.s.c.ZkTestServer waitForServerUp: 127.0.0.1:36087
   [junit4]   2> 3391799 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:36087
   [junit4]   2> 3391799 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 36087
   [junit4]   2> 3391800 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 3391803 INFO  (zkConnectionManagerCallback-15769-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3391803 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 3391806 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 3391806 INFO  (zkConnectionManagerCallback-15771-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3391806 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 3391807 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 3391808 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 3391809 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 3391809 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 3391810 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 3391811 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 3391811 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 3391812 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 3391812 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 3391813 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 3391814 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 3391814 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
   [junit4]   2> 3391888 WARN  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 3391888 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 3391888 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 3391888 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.e.j.s.Server jetty-9.4.19.v20190610; built: 2019-06-10T16:30:51.723Z; git: afcf563148970e98786327af5e07c261fda175d3; jvm 1.8.0_191-b12
   [junit4]   2> 3391888 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 3391888 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 3391889 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 3391889 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@a926353{/,null,AVAILABLE}
   [junit4]   2> 3391889 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.e.j.s.AbstractConnector Started ServerConnector@53551b72{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:34343}
   [junit4]   2> 3391889 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.e.j.s.Server Started @3391916ms
   [junit4]   2> 3391889 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.api.collections.ShardSplitTest_32A32D8993FC5BEE-001/tempDir-001/control/data, hostContext=/, hostPort=34343, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.api.collections.ShardSplitTest_32A32D8993FC5BEE-001/control-001/cores}
   [junit4]   2> 3391889 ERROR (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 3391889 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 3391889 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solrâ„¢ version 8.5.0
   [junit4]   2> 3391889 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 3391889 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 3391889 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-12-13T07:41:02.438Z
   [junit4]   2> 3391890 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 3391891 INFO  (zkConnectionManagerCallback-15773-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3391891 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 3391992 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 3391992 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.api.collections.ShardSplitTest_32A32D8993FC5BEE-001/control-001/solr.xml
   [junit4]   2> 3391995 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 3391995 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 3391997 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1cb15233, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 3392065 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
   [junit4]   2> 3392066 WARN  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@1aef8a06[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 3392066 WARN  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@1aef8a06[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 3392068 WARN  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@f5f9746[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 3392068 WARN  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@f5f9746[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 3392069 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36087/solr
   [junit4]   2> 3392070 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 3392071 INFO  (zkConnectionManagerCallback-15780-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3392071 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 3392172 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [n:127.0.0.1:34343_     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 3392173 INFO  (zkConnectionManagerCallback-15782-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3392173 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [n:127.0.0.1:34343_     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 3392213 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [n:127.0.0.1:34343_     ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:34343_
   [junit4]   2> 3392214 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [n:127.0.0.1:34343_     ] o.a.s.c.Overseer Overseer (id=72886261073182724-127.0.0.1:34343_-n_0000000000) starting
   [junit4]   2> 3392217 DEBUG (OverseerCollectionConfigSetProcessor-72886261073182724-127.0.0.1:34343_-n_0000000000) [n:127.0.0.1:34343_     ] o.a.s.c.OverseerTaskProcessor Process current queue of overseer operations
   [junit4]   2> 3392217 INFO  (OverseerStateUpdate-72886261073182724-127.0.0.1:34343_-n_0000000000) [n:127.0.0.1:34343_     ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:34343_
   [junit4]   2> 3392217 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [n:127.0.0.1:34343_     ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:34343_
   [junit4]   2> 3392220 DEBUG (OverseerCollectionConfigSetProcessor-72886261073182724-127.0.0.1:34343_-n_0000000000) [n:127.0.0.1:34343_     ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0 #Completed tasks: 0
   [junit4]   2> 3392220 DEBUG (OverseerCollectionConfigSetProcessor-72886261073182724-127.0.0.1:34343_-n_0000000000) [n:127.0.0.1:34343_     ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 3392220 INFO  (OverseerStateUpdate-72886261073182724-127.0.0.1:34343_-n_0000000000) [n:127.0.0.1:34343_     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 3392220 DEBUG (OverseerCollectionConfigSetProcessor-72886261073182724-127.0.0.1:34343_-n_0000000000) [n:127.0.0.1:34343_     ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 3392221 DEBUG (OverseerCollectionConfigSetProcessor-72886261073182724-127.0.0.1:34343_-n_0000000000) [n:127.0.0.1:34343_     ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 3392221 INFO  (OverseerCollectionConfigSetProcessor-72886261073182724-127.0.0.1:34343_-n_0000000000) [n:127.0.0.1:34343_     ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 3392221 WARN  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [n:127.0.0.1:34343_     ] o.a.s.c.CoreContainer Not all security plugins configured!  authentication=disabled authorization=disabled.  Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external.  See https://s.apache.org/solrsecurity for more info
   [junit4]   2> 3392233 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [n:127.0.0.1:34343_     ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 3392247 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [n:127.0.0.1:34343_     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1cb15233
   [junit4]   2> 3392252 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [n:127.0.0.1:34343_     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1cb15233
   [junit4]   2> 3392252 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [n:127.0.0.1:34343_     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1cb15233
   [junit4]   2> 3392253 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [n:127.0.0.1:34343_     ] o.a.s.p.PackageLoader /packages.json updated to version -1
   [junit4]   2> 3392254 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [n:127.0.0.1:34343_     ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.api.collections.ShardSplitTest_32A32D8993FC5BEE-001/control-001/cores
   [junit4]   2> 3392267 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 3392268 INFO  (zkConnectionManagerCallback-15791-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3392268 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 3392269 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 3392269 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:36087/solr ready
   [junit4]   2> 3392270 INFO  (qtp668097043-289555) [n:127.0.0.1:34343_     ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:34343_&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 3392272 DEBUG (OverseerCollectionConfigSetProcessor-72886261073182724-127.0.0.1:34343_-n_0000000000) [n:127.0.0.1:34343_     ] o.a.s.c.OverseerTaskProcessor Got 1 tasks from work-queue : [[org.apache.solr.cloud.OverseerTaskQueue$QueueEvent@2e684ecf]]
   [junit4]   2> 3392272 DEBUG (OverseerCollectionConfigSetProcessor-72886261073182724-127.0.0.1:34343_-n_0000000000) [n:127.0.0.1:34343_     ] o.a.s.c.OverseerTaskProcessor Marked task [/overseer/collection-queue-work/qn-0000000000] as running
   [junit4]   2> 3392272 DEBUG (OverseerCollectionConfigSetProcessor-72886261073182724-127.0.0.1:34343_-n_0000000000) [n:127.0.0.1:34343_     ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Get the message id:/overseer/collection-queue-work/qn-0000000000 message:{
   [junit4]   2>   "name":"control_collection",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "collection.configName":"conf1",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "createNodeSet":"127.0.0.1:34343_",
   [junit4]   2>   "nrtReplicas":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 3392272 DEBUG (OverseerThreadFactory-14067-thread-1-processing-n:127.0.0.1:34343_) [n:127.0.0.1:34343_     ] o.a.s.c.OverseerTaskProcessor Runner processing /overseer/collection-queue-work/qn-0000000000
   [junit4]   2> 3392272 DEBUG (OverseerThreadFactory-14067-thread-1-processing-n:127.0.0.1:34343_) [n:127.0.0.1:34343_     ] o.a.s.c.a.c.OverseerCollectionMessageHandler OverseerCollectionMessageHandler.processMessage : create , {
   [junit4]   2>   "name":"control_collection",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "collection.configName":"conf1",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "createNodeSet":"127.0.0.1:34343_",
   [junit4]   2>   "nrtReplicas":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 3392272 DEBUG (OverseerCollectionConfigSetProcessor-72886261073182724-127.0.0.1:34343_-n_0000000000) [n:127.0.0.1:34343_     ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 1 #Completed tasks: 0
   [junit4]   2> 3392272 DEBUG (OverseerCollectionConfigSetProcessor-72886261073182724-127.0.0.1:34343_-n_0000000000) [n:127.0.0.1:34343_     ] o.a.s.c.OverseerTaskProcessor RunningTasks: [/overseer/collection-queue-work/qn-0000000000]
   [junit4]   2> 3392272 DEBUG (OverseerCollectionConfigSetProcessor-72886261073182724-127.0.0.1:34343_-n_0000000000) [n:127.0.0.1:34343_     ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 3392272 DEBUG (OverseerCollectionConfigSetProcessor-72886261073182724-127.0.0.1:34343_-n_0000000000) [n:127.0.0.1:34343_     ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 3392272 INFO  (OverseerCollectionConfigSetProcessor-72886261073182724-127.0.0.1:34343_-n_0000000000) [n:127.0.0.1:34343_     ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: [/overseer/collection-queue-work/qn-0000000000]
   [junit4]   2> 3392272 INFO  (OverseerThreadFactory-14067-thread-1-processing-n:127.0.0.1:34343_) [n:127.0.0.1:34343_     ] o.a.s.c.a.c.CreateCollectionCmd Create collection control_collection
   [junit4]   2> 3392273 DEBUG (OverseerThreadFactory-14067-thread-1-processing-n:127.0.0.1:34343_) [n:127.0.0.1:34343_     ] o.a.s.c.a.c.OverseerCollectionMessageHandler creating collections conf node /collections/control_collection 
   [junit4]   2> 3392273 DEBUG (OverseerThreadFactory-14067-thread-1-processing-n:127.0.0.1:34343_) [n:127.0.0.1:34343_     ] o.a.s.c.a.c.CreateCollectionCmd Check for collection zkNode:control_collection
   [junit4]   2> 3392273 DEBUG (OverseerThreadFactory-14067-thread-1-processing-n:127.0.0.1:34343_) [n:127.0.0.1:34343_     ] o.a.s.c.a.c.CreateCollectionCmd Collection zkNode exists
   [junit4]   2> 3392274 DEBUG (OverseerStateUpdate-72886261073182724-127.0.0.1:34343_-n_0000000000) [n:127.0.0.1:34343_     ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "name":"control_collection",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "collection.configName":"conf1",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "createNodeSet":"127.0.0.1:34343_",
   [junit4]   2>   "nrtReplicas":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "operation":"create"} current state version: 0
   [junit4]   2> 3392274 DEBUG (OverseerStateUpdate-72886261073182724-127.0.0.1:34343_-n_0000000000) [n:127.0.0.1:34343_     ] o.a.s.c.o.ClusterStateMutator building a new cName: control_collection
   [junit4]   2> 3392274 DEBUG (OverseerStateUpdate-72886261073182724-127.0.0.1:34343_-n_0000000000) [n:127.0.0.1:34343_     ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/control_collection/state.json
   [junit4]   2> 3392375 DEBUG (OverseerThreadFactory-14067-thread-1-processing-n:127.0.0.1:34343_) [n:127.0.0.1:34343_     ] o.a.s.c.a.c.CreateCollectionCmd Creating SolrCores for new collection control_collection, shardNames [shard1] , message : {
   [junit4]   2>   "name":"control_collection",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "collection.configName":"conf1",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "createNodeSet":"127.0.0.1:34343_",
   [junit4]   2>   "nrtReplicas":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 3392376 DEBUG (OverseerThreadFactory-14067-thread-1-processing-n:127.0.0.1:34343_) [n:127.0.0.1:34343_     ] o.a.s.c.a.c.CreateCollectionCmd Creating core control_collection_shard1_replica_n1 as part of shard shard1 of collection control_collection on 127.0.0.1:34343_
   [junit4]   2> 3392377 INFO  (qtp668097043-289557) [n:127.0.0.1:34343_    x:control_collection_shard1_replica_n1 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 3392377 INFO  (qtp668097043-289557) [n:127.0.0.1:34343_    x:control_collection_shard1_replica_n1 ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 3392379 DEBUG (OverseerStateUpdate-72886261073182724-127.0.0.1:34343_-n_0000000000) [n:127.0.0.1:34343_     ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "core":"control_collection_shard1_replica_n1",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"http://127.0.0.1:34343",
   [junit4]   2>   "node_name":"127.0.0.1:34343_",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "collection":"control_collection",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "operation":"state"} current state version: 0
   [junit4]   2> 3392379 DEBUG (OverseerStateUpdate-72886261073182724-127.0.0.1:34343_-n_0000000000) [n:127.0.0.1:34343_     ] o.a.s.c.o.ReplicaMutator Update state numShards=1 message={
   [junit4]   2>   "core":"control_collection_shard1_replica_n1",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"http://127.0.0.1:34343",
   [junit4]   2>   "node_name":"127.0.0.1:34343_",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "collection":"control_collection",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "operation":"state"}
   [junit4]   2> 3392381 DEBUG (OverseerStateUpdate-72886261073182724-127.0.0.1:34343_-n_0000000000) [n:127.0.0.1:34343_     ] o.a.s.c.o.ReplicaMutator Will update state for replica: core_node2:{
   [junit4]   2>   "core":"control_collection_shard1_replica_n1",
   [junit4]   2>   "base_url":"http://127.0.0.1:34343",
   [junit4]   2>   "node_name":"127.0.0.1:34343_",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "type":"NRT"}
   [junit4]   2> 3392381 DEBUG (OverseerStateUpdate-72886261073182724-127.0.0.1:34343_-n_0000000000) [n:127.0.0.1:34343_     ] o.a.s.c.o.ReplicaMutator Collection is now: DocCollection(control_collection//collections/control_collection/state.json/0)={
   [junit4]   2>   "pullReplicas":"0",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "router":{"name":"compositeId"},
   [junit4]   2>   "maxShardsPerNode":"1",
   [junit4]   2>   "autoAddReplicas":"false",
   [junit4]   2>   "nrtReplicas":"1",
   [junit4]   2>   "tlogReplicas":"0",
   [junit4]   2>   "shards":{"shard1":{
   [junit4]   2>       "range":"80000000-7fffffff",
   [junit4]   2>       "state":"active",
   [junit4]   2>       "replicas":{"core_node2":{
   [junit4]   2>           "core":"control_collection_shard1_replica_n1",
   [junit4]   2>           "base_url":"http://127.0.0.1:34343",
   [junit4]   2>           "node_name":"127.0.0.1:34343_",
   [junit4]   2>           "state":"down",
   [junit4]   2>           "type":"NRT"}}}}}
   [junit4]   2> 3392482 DEBUG (OverseerStateUpdate-72886261073182724-127.0.0.1:34343_-n_0000000000) [n:127.0.0.1:34343_     ] o.a.s.c.o.ZkStateWriter going to update_collection /collections/control_collection/state.json version: 0
   [junit4]   2> 3393387 INFO  (qtp668097043-289557) [n:127.0.0.1:34343_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.5.0
   [junit4]   2> 3393414 INFO  (qtp668097043-289557) [n:127.0.0.1:34343_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
   [junit4]   2> 3393472 INFO  (qtp668097043-289557) [n:127.0.0.1:34343_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 3393483 INFO  (qtp668097043-289557) [n:127.0.0.1:34343_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
   [junit4]   2> 3393483 INFO  (qtp668097043-289557) [n:127.0.0.1:34343_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1cb15233
   [junit4]   2> 3393483 INFO  (qtp668097043-289557) [n:127.0.0.1:34343_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.api.collections.ShardSplitTest_32A32D8993FC5BEE-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.api.collections.ShardSplitTest_32A32D8993FC5BEE-001/control-001/cores/control_collection_shard1_replica_n1/data/]
   [junit4]   2> 3393485 INFO  (qtp668097043-289557) [n:127.0.0.1:34343_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=10, maxMergeAtOnceExplicit=24, maxMergedSegmentMB=30.103515625, floorSegmentMB=0.564453125, forceMergeDeletesPctAllowed=27.353977665861805, segmentsPerTier=43.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5566435627349643, deletesPctAllowed=43.770737289776285
   [junit4]   2> 3393493 WARN  (qtp668097043-289557) [n:127.0.0.1:34343_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 3393529 INFO  (qtp668097043-289557) [n:127.0.0.1:34343_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 3393529 INFO  (qtp668097043-289557) [n:127.0.0.1:34343_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 3393530 INFO  (qtp668097043-289557) [n:127.0.0.1:34343_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 3393530 INFO  (qtp668097043-289557) [n:127.0.0.1:34343_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 3393531 INFO  (qtp668097043-289557) [n:127.0.0.1:34343_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@58ca5a9b
   [junit4]   2> 3393531 INFO  (qtp668097043-289557) [n:127.0.0.1:34343_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.s.SolrIndexSearcher Opening [Searcher@1b3d5e8a[control_collection_shard1_replica_n1] main]
   [junit4]   2> 3393532 INFO  (qtp668097043-289557) [n:127.0.0.1:34343_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 3393534 INFO  (qtp668097043-289557) [n:127.0.0.1:34343_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 3393535 INFO  (qtp668097043-289557) [n:127.0.0.1:34343_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 3393535 INFO  (qtp668097043-289557) [n:127.0.0.1:34343_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1652789465929744384
   [junit4]   2> 3393537 INFO  (searcherExecutor-14072-thread-1-processing-n:127.0.0.1:34343_ x:control_collection_shard1_replica_n1 c:control_collection s:shard1) [n:127.0.0.1:34343_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@1b3d5e8a[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 3393539 INFO  (qtp668097043-289557) [n:127.0.0.1:34343_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.ZkShardTerms Successful update of terms at /collections/control_collection/terms/shard1 to Terms{values={core_node2=0}, version=0}
   [junit4]   2> 3393539 INFO  (qtp668097043-289557) [n:127.0.0.1:34343_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/control_collection/leaders/shard1
   [junit4]   2> 3393541 INFO  (qtp668097043-289557) [n:127.0.0.1:34343_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 3393541 INFO  (qtp668097043-289557) [n:127.0.0.1:34343_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 3393541 INFO  (qtp668097043-289557) [n:127.0.0.1:34343_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:34343/control_collection_shard1_replica_n1/
   [junit4]   2> 3393541 INFO  (qtp668097043-289557) [n:127.0.0.1:34343_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 3393542 INFO  (qtp668097043-289557) [n:127.0.0.1:34343_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.SyncStrategy http://127.0.0.1:34343/control_collection_shard1_replica_n1/ has no replicas
   [junit4]   2> 3393542 INFO  (qtp668097043-289557) [n:127.0.0.1:34343_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/control_collection/leaders/shard1/leader after winning as /collections/control_collection/leader_elect/shard1/election/72886261073182724-core_node2-n_0000000000
   [junit4]   2> 3393544 INFO  (qtp668097043-289557) [n:127.0.0.1:34343_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:34343/control_collection_shard1_replica_n1/ shard1
   [junit4]   2> 3393544 DEBUG (OverseerStateUpdate-72886261073182724-127.0.0.1:34343_-n_0000000000) [n:127.0.0.1:34343_     ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "operation":"leader",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "collection":"control_collection",
   [junit4]   2>   "base_url":"http://127.0.0.1:34343",
   [junit4]   2>   "core":"control_collection_shard1_replica_n1",
   [junit4]   2>   "state":"active"} current state version: 0
   [junit4]   2> 3393645 DEBUG (OverseerStateUpdate-72886261073182724-127.0.0.1:34343_-n_0000000000) [n:127.0.0.1:34343_     ] o.a.s.c.o.ZkStateWriter going to update_collection /collections/control_collection/state.json version: 1
   [junit4]   2> 3393645 INFO  (zkCallback-15781-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 3393646 INFO  (qtp668097043-289557) [n:127.0.0.1:34343_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 3393647 INFO  (qtp668097043-289557) [n:127.0.0.1:34343_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1270
   [junit4]   2> 3393647 INFO  (zkCallback-15781-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 3393647 DEBUG (OverseerThreadFactory-14067-thread-1-processing-n:127.0.0.1:34343_) [n:127.0.0.1:34343_     ] o.a.s.c.a.c.CreateCollectionCmd Finished create command on all shards for collection: control_collection
   [junit4]   2> 3393647 DEBUG (OverseerThreadFactory-14067-thread-1-processing-n:127.0.0.1:34343_) [n:127.0.0.1:34343_     ] o.a.s.c.OverseerTaskProcessor Completed task:[/overseer/collection-queue-work/qn-0000000000]
   [junit4]   2> 3393648 DEBUG (OverseerStateUpdate-72886261073182724-127.0.0.1:34343_-n_0000000000) [n:127.0.0.1:34343_     ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "core":"control_collection_shard1_replica_n1",
   [junit4]   2>   "core_node_name":"core_node2",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"http://127.0.0.1:34343",
   [junit4]   2>   "node_name":"127.0.0.1:34343_",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "state":"active",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "collection":"control_collection",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "operation":"state"} current state version: 0
   [junit4]   2> 3393648 DEBUG (OverseerStateUpdate-72886261073182724-127.0.0.1:34343_-n_0000000000) [n:127.0.0.1:34343_     ] o.a.s.c.o.ReplicaMutator Update state numShards=1 message={
   [junit4]   2>   "core":"control_collection_shard1_replica_n1",
   [junit4]   2>   "core_node_name":"core_node2",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"http://127.0.0.1:34343",
   [junit4]   2>   "node_name":"127.0.0.1:34343_",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "state":"active",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "collection":"control_collection",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "operation":"state"}
   [junit4]   2> 3393648 DEBUG (OverseerStateUpdate-72886261073182724-127.0.0.1:34343_-n_0000000000) [n:127.0.0.1:34343_     ] o.a.s.c.o.ReplicaMutator Will update state for replica: core_node2:{
   [junit4]   2>   "core":"control_collection_shard1_replica_n1",
   [junit4]   2>   "base_url":"http://127.0.0.1:34343",
   [junit4]   2>   "node_name":"127.0.0.1:34343_",
   [junit4]   2>   "state":"active",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "leader":"true"}
   [junit4]   2> 3393648 DEBUG (OverseerStateUpdate-72886261073182724-127.0.0.1:34343_-n_0000000000) [n:127.0.0.1:34343_     ] o.a.s.c.o.ReplicaMutator Collection is now: DocCollection(control_collection//collections/control_collection/state.json/2)={
   [junit4]   2>   "pullReplicas":"0",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "router":{"name":"compositeId"},
   [junit4]   2>   "maxShardsPerNode":"1",
   [junit4]   2>   "autoAddReplicas":"false",
   [junit4]   2>   "nrtReplicas":"1",
   [junit4]   2>   "tlogReplicas":"0",
   [junit4]   2>   "shards":{"shard1":{
   [junit4]   2>       "range":"80000000-7fffffff",
   [junit4]   2>       "state":"active",
   [junit4]   2>       "replicas":{"core_node2":{
   [junit4]   2>           "core":"control_collection_shard1_replica_n1",
   [junit4]   2>           "base_url":"http://127.0.0.1:34343",
   [junit4]   2>           "node_name":"127.0.0.1:34343_",
   [junit4]   2>           "state":"active",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "leader":"true"}}}}}
   [junit4]   2> 3393648 DEBUG (OverseerThreadFactory-14067-thread-1-processing-n:127.0.0.1:34343_) [n:127.0.0.1:34343_     ] o.a.s.c.OverseerTaskProcessor Marked task [/overseer/collection-queue-work/qn-0000000000] as completed.
   [junit4]   2> 3393648 DEBUG (OverseerThreadFactory-14067-thread-1-processing-n:127.0.0.1:34343_) [n:127.0.0.1:34343_     ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 3393648 DEBUG (OverseerThreadFactory-14067-thread-1-processing-n:127.0.0.1:34343_) [n:127.0.0.1:34343_     ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 3393648 DEBUG (OverseerThreadFactory-14067-thread-1-processing-n:127.0.0.1:34343_) [n:127.0.0.1:34343_     ] o.a.s.c.OverseerTaskProcessor CompletedTasks: [/overseer/collection-queue-work/qn-0000000000]
   [junit4]   2> 3393648 INFO  (OverseerThreadFactory-14067-thread-1-processing-n:127.0.0.1:34343_) [n:127.0.0.1:34343_     ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: [/overseer/collection-queue-work/qn-0000000000]
   [junit4]   2> 3393648 DEBUG (OverseerThreadFactory-14067-thread-1-processing-n:127.0.0.1:34343_) [n:127.0.0.1:34343_     ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Message id:/overseer/collection-queue-work/qn-0000000000 complete, response:{success={127.0.0.1:34343_={responseHeader={status=0,QTime=1270},core=control_collection_shard1_replica_n1}}}
   [junit4]   2> 3393650 INFO  (qtp668097043-289555) [n:127.0.0.1:34343_     ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
   [junit4]   2> 3393749 DEBUG (OverseerStateUpdate-72886261073182724-127.0.0.1:34343_-n_0000000000) [n:127.0.0.1:34343_     ] o.a.s.c.o.ZkStateWriter going to update_collection /collections/control_collection/state.json version: 2
   [junit4]   2> 3393749 INFO  (zkCallback-15781-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 3393749 INFO  (zkCallback-15781-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 3393749 INFO  (zkCallback-15781-thread-3) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 3393749 INFO  (qtp668097043-289555) [n:127.0.0.1:34343_     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:34343_&wt=javabin&version=2} status=0 QTime=1479
   [junit4]   2> 3393750 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.a.s.c.AbstractFullDistribZkTestBase Waiting to see 1 active replicas in collection: control_collection
   [junit4]   2> 3393853 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 3393854 INFO  (zkConnectionManagerCallback-15797-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3393854 INFO  (TEST-ShardSplitTest.testSplitStaticIndexReplicationLink-seed#[32A32D8993FC5BEE]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 339

[...truncated too long message...]

.Thread.run(Thread.java:748)
   [junit4]    >    2) Thread[id=293261, name=qtp1683749906-293261, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    3) Thread[id=293258, name=qtp1683749906-293258, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    4) Thread[id=293264, name=qtp1683749906-293264, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    5) Thread[id=293342, name=Connector-Scheduler-60646441, state=WAITING, group=TGRP-ShardSplitTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    6) Thread[id=293263, name=qtp1683749906-293263, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    7) Thread[id=293259, name=qtp1683749906-293259, state=RUNNABLE, group=TGRP-ShardSplitTest]
   [junit4]    >         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
   [junit4]    >         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
   [junit4]    >         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
   [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$$Lambda$59/1646479797.run(Unknown Source)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    8) Thread[id=293257, name=qtp1683749906-293257, state=RUNNABLE, group=TGRP-ShardSplitTest]
   [junit4]    >         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
   [junit4]    >         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
   [junit4]    >         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
   [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$$Lambda$59/1646479797.run(Unknown Source)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    9) Thread[id=293262, name=qtp1683749906-293262, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   10) Thread[id=293265, name=Session-HouseKeeper-68180a62, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([32A32D8993FC5BEE]:0)
   [junit4] Completed [554/889 (1!)] on J0 in 607.62s, 11 tests, 3 errors <<< FAILURES!

[...truncated 53947 lines...]

[JENKINS] Lucene-Solr-NightlyTests-8.x - Build # 294 - Still Unstable

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-8.x/294/

1 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.autoscaling.SearchRateTriggerIntegrationTest

Error Message:
Clean up static fields (in @AfterClass?) and null them, your test still has references to classes of which the sizes cannot be measured due to security restrictions or Java 9 module encapsulation:   - private static org.apache.solr.client.solrj.cloud.SolrCloudManager org.apache.solr.cloud.autoscaling.SearchRateTriggerIntegrationTest.cloudManager

Stack Trace:
junit.framework.AssertionFailedError: Clean up static fields (in @AfterClass?) and null them, your test still has references to classes of which the sizes cannot be measured due to security restrictions or Java 9 module encapsulation:
  - private static org.apache.solr.client.solrj.cloud.SolrCloudManager org.apache.solr.cloud.autoscaling.SearchRateTriggerIntegrationTest.cloudManager
	at __randomizedtesting.SeedInfo.seed([9BD8837E45B27770]:0)
	at com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule$1.afterAlways(StaticFieldsInvariantRule.java:146)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.security.AccessControlException: access denied ("java.lang.RuntimePermission" "accessClassInPackage.sun.nio.fs")
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.security.AccessController.checkPermission(AccessController.java:884)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkPackageAccess(SecurityManager.java:1564)
	at java.lang.Class.checkPackageAccess(Class.java:2372)
	at java.lang.Class.checkMemberAccess(Class.java:2351)
	at java.lang.Class.getDeclaredFields(Class.java:1915)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator$2.run(RamUsageEstimator.java:585)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator$2.run(RamUsageEstimator.java:582)
	at java.security.AccessController.doPrivileged(Native Method)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.createCacheEntry(RamUsageEstimator.java:582)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.measureSizeOf(RamUsageEstimator.java:545)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.sizeOfAll(RamUsageEstimator.java:387)
	at com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule$1.afterAlways(StaticFieldsInvariantRule.java:129)
	... 10 more




Build Log:
[...truncated 15708 lines...]
   [junit4] Suite: org.apache.solr.cloud.autoscaling.SearchRateTriggerIntegrationTest
   [junit4]   2> 6020202 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[9BD8837E45B27770]-worker) [     ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 6020203 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[9BD8837E45B27770]-worker) [     ] o.a.s.SolrTestCaseJ4 Created dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.SearchRateTriggerIntegrationTest_9BD8837E45B27770-001/data-dir-231-001
   [junit4]   2> 6020203 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[9BD8837E45B27770]-worker) [     ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 6020205 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[9BD8837E45B27770]-worker) [     ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 6020205 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[9BD8837E45B27770]-worker) [     ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 5 servers in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.SearchRateTriggerIntegrationTest_9BD8837E45B27770-001/tempDir-001
   [junit4]   2> 6020205 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[9BD8837E45B27770]-worker) [     ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 6020206 INFO  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 6020206 INFO  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 6020306 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[9BD8837E45B27770]-worker) [     ] o.a.s.c.ZkTestServer start zk server on port:45263
   [junit4]   2> 6020306 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[9BD8837E45B27770]-worker) [     ] o.a.s.c.ZkTestServer waitForServerUp: 127.0.0.1:45263
   [junit4]   2> 6020306 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[9BD8837E45B27770]-worker) [     ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:45263
   [junit4]   2> 6020306 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[9BD8837E45B27770]-worker) [     ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 45263
   [junit4]   2> 6020308 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[9BD8837E45B27770]-worker) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 6020310 INFO  (zkConnectionManagerCallback-20678-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 6020311 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[9BD8837E45B27770]-worker) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 6020314 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[9BD8837E45B27770]-worker) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 6020314 INFO  (zkConnectionManagerCallback-20680-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 6020314 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[9BD8837E45B27770]-worker) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 6020315 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[9BD8837E45B27770]-worker) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 6020316 INFO  (zkConnectionManagerCallback-20682-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 6020316 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[9BD8837E45B27770]-worker) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 6020425 WARN  (jetty-launcher-20683-thread-3) [     ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 6020425 WARN  (jetty-launcher-20683-thread-5) [     ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 6020425 WARN  (jetty-launcher-20683-thread-4) [     ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 6020426 INFO  (jetty-launcher-20683-thread-4) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 6020426 INFO  (jetty-launcher-20683-thread-5) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 6020426 INFO  (jetty-launcher-20683-thread-3) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 6020426 INFO  (jetty-launcher-20683-thread-3) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 6020426 INFO  (jetty-launcher-20683-thread-5) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 6020426 INFO  (jetty-launcher-20683-thread-4) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 6020426 WARN  (jetty-launcher-20683-thread-2) [     ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 6020426 INFO  (jetty-launcher-20683-thread-5) [     ] o.e.j.s.Server jetty-9.4.19.v20190610; built: 2019-06-10T16:30:51.723Z; git: afcf563148970e98786327af5e07c261fda175d3; jvm 1.8.0_191-b12
   [junit4]   2> 6020426 INFO  (jetty-launcher-20683-thread-3) [     ] o.e.j.s.Server jetty-9.4.19.v20190610; built: 2019-06-10T16:30:51.723Z; git: afcf563148970e98786327af5e07c261fda175d3; jvm 1.8.0_191-b12
   [junit4]   2> 6020426 INFO  (jetty-launcher-20683-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 6020426 INFO  (jetty-launcher-20683-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 6020426 WARN  (jetty-launcher-20683-thread-1) [     ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 6020426 INFO  (jetty-launcher-20683-thread-2) [     ] o.e.j.s.Server jetty-9.4.19.v20190610; built: 2019-06-10T16:30:51.723Z; git: afcf563148970e98786327af5e07c261fda175d3; jvm 1.8.0_191-b12
   [junit4]   2> 6020426 INFO  (jetty-launcher-20683-thread-4) [     ] o.e.j.s.Server jetty-9.4.19.v20190610; built: 2019-06-10T16:30:51.723Z; git: afcf563148970e98786327af5e07c261fda175d3; jvm 1.8.0_191-b12
   [junit4]   2> 6020426 INFO  (jetty-launcher-20683-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 6020426 INFO  (jetty-launcher-20683-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 6020426 INFO  (jetty-launcher-20683-thread-1) [     ] o.e.j.s.Server jetty-9.4.19.v20190610; built: 2019-06-10T16:30:51.723Z; git: afcf563148970e98786327af5e07c261fda175d3; jvm 1.8.0_191-b12
   [junit4]   2> 6020427 INFO  (jetty-launcher-20683-thread-1) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 6020427 INFO  (jetty-launcher-20683-thread-1) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 6020427 INFO  (jetty-launcher-20683-thread-1) [     ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 6020427 INFO  (jetty-launcher-20683-thread-1) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3150ff29{/solr,null,AVAILABLE}
   [junit4]   2> 6020427 INFO  (jetty-launcher-20683-thread-1) [     ] o.e.j.s.AbstractConnector Started ServerConnector@9f055b2{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:42540}
   [junit4]   2> 6020427 INFO  (jetty-launcher-20683-thread-1) [     ] o.e.j.s.Server Started @6020464ms
   [junit4]   2> 6020427 INFO  (jetty-launcher-20683-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=42540}
   [junit4]   2> 6020428 ERROR (jetty-launcher-20683-thread-1) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 6020428 INFO  (jetty-launcher-20683-thread-1) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 6020428 INFO  (jetty-launcher-20683-thread-1) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solrâ„¢ version 8.4.0
   [junit4]   2> 6020428 INFO  (jetty-launcher-20683-thread-1) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 6020428 INFO  (jetty-launcher-20683-thread-1) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 6020428 INFO  (jetty-launcher-20683-thread-1) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-12-12T10:11:16.780Z
   [junit4]   2> 6020429 INFO  (jetty-launcher-20683-thread-1) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 6020435 INFO  (zkConnectionManagerCallback-20685-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 6020435 INFO  (jetty-launcher-20683-thread-1) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 6020436 INFO  (jetty-launcher-20683-thread-1) [     ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 6020444 INFO  (jetty-launcher-20683-thread-3) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 6020444 INFO  (jetty-launcher-20683-thread-3) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 6020444 INFO  (jetty-launcher-20683-thread-3) [     ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 6020444 INFO  (jetty-launcher-20683-thread-4) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 6020444 INFO  (jetty-launcher-20683-thread-4) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 6020444 INFO  (jetty-launcher-20683-thread-4) [     ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 6020444 INFO  (jetty-launcher-20683-thread-4) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@599a8b14{/solr,null,AVAILABLE}
   [junit4]   2> 6020445 INFO  (jetty-launcher-20683-thread-3) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@53a6340c{/solr,null,AVAILABLE}
   [junit4]   2> 6020445 INFO  (jetty-launcher-20683-thread-5) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 6020445 INFO  (jetty-launcher-20683-thread-5) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 6020445 INFO  (jetty-launcher-20683-thread-5) [     ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 6020445 INFO  (jetty-launcher-20683-thread-2) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 6020445 INFO  (jetty-launcher-20683-thread-2) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 6020445 INFO  (jetty-launcher-20683-thread-2) [     ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 6020445 INFO  (jetty-launcher-20683-thread-5) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@75641db4{/solr,null,AVAILABLE}
   [junit4]   2> 6020445 INFO  (jetty-launcher-20683-thread-3) [     ] o.e.j.s.AbstractConnector Started ServerConnector@32f0d27b{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:39700}
   [junit4]   2> 6020445 INFO  (jetty-launcher-20683-thread-3) [     ] o.e.j.s.Server Started @6020481ms
   [junit4]   2> 6020445 INFO  (jetty-launcher-20683-thread-2) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@325fac69{/solr,null,AVAILABLE}
   [junit4]   2> 6020445 INFO  (jetty-launcher-20683-thread-3) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=39700}
   [junit4]   2> 6020445 ERROR (jetty-launcher-20683-thread-3) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 6020445 INFO  (jetty-launcher-20683-thread-3) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 6020445 INFO  (jetty-launcher-20683-thread-5) [     ] o.e.j.s.AbstractConnector Started ServerConnector@2477c011{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:45317}
   [junit4]   2> 6020445 INFO  (jetty-launcher-20683-thread-5) [     ] o.e.j.s.Server Started @6020481ms
   [junit4]   2> 6020445 INFO  (jetty-launcher-20683-thread-5) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=45317}
   [junit4]   2> 6020445 INFO  (jetty-launcher-20683-thread-3) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solrâ„¢ version 8.4.0
   [junit4]   2> 6020445 INFO  (jetty-launcher-20683-thread-4) [     ] o.e.j.s.AbstractConnector Started ServerConnector@6646e54d{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:33174}
   [junit4]   2> 6020445 INFO  (jetty-launcher-20683-thread-4) [     ] o.e.j.s.Server Started @6020482ms
   [junit4]   2> 6020445 INFO  (jetty-launcher-20683-thread-3) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 6020445 INFO  (jetty-launcher-20683-thread-3) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 6020445 INFO  (jetty-launcher-20683-thread-4) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=33174}
   [junit4]   2> 6020445 INFO  (jetty-launcher-20683-thread-3) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-12-12T10:11:16.797Z
   [junit4]   2> 6020446 ERROR (jetty-launcher-20683-thread-5) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 6020446 INFO  (jetty-launcher-20683-thread-5) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 6020446 ERROR (jetty-launcher-20683-thread-4) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 6020446 INFO  (jetty-launcher-20683-thread-4) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 6020446 INFO  (jetty-launcher-20683-thread-5) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solrâ„¢ version 8.4.0
   [junit4]   2> 6020446 INFO  (jetty-launcher-20683-thread-5) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 6020446 INFO  (jetty-launcher-20683-thread-5) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 6020446 INFO  (jetty-launcher-20683-thread-5) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-12-12T10:11:16.798Z
   [junit4]   2> 6020446 INFO  (jetty-launcher-20683-thread-4) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solrâ„¢ version 8.4.0
   [junit4]   2> 6020446 INFO  (jetty-launcher-20683-thread-4) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 6020446 INFO  (jetty-launcher-20683-thread-4) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 6020446 INFO  (jetty-launcher-20683-thread-4) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-12-12T10:11:16.798Z
   [junit4]   2> 6020447 INFO  (jetty-launcher-20683-thread-3) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 6020447 INFO  (jetty-launcher-20683-thread-5) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 6020447 INFO  (jetty-launcher-20683-thread-2) [     ] o.e.j.s.AbstractConnector Started ServerConnector@14f912ac{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:37817}
   [junit4]   2> 6020447 INFO  (jetty-launcher-20683-thread-2) [     ] o.e.j.s.Server Started @6020483ms
   [junit4]   2> 6020447 INFO  (jetty-launcher-20683-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=37817}
   [junit4]   2> 6020447 ERROR (jetty-launcher-20683-thread-2) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 6020447 INFO  (jetty-launcher-20683-thread-2) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 6020447 INFO  (jetty-launcher-20683-thread-2) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solrâ„¢ version 8.4.0
   [junit4]   2> 6020447 INFO  (jetty-launcher-20683-thread-2) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 6020447 INFO  (jetty-launcher-20683-thread-2) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 6020447 INFO  (jetty-launcher-20683-thread-2) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-12-12T10:11:16.799Z
   [junit4]   2> 6020447 INFO  (zkConnectionManagerCallback-20687-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 6020447 INFO  (jetty-launcher-20683-thread-3) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 6020448 INFO  (jetty-launcher-20683-thread-2) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 6020449 INFO  (jetty-launcher-20683-thread-4) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 6020449 INFO  (zkConnectionManagerCallback-20689-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 6020449 INFO  (jetty-launcher-20683-thread-5) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 6020449 INFO  (zkConnectionManagerCallback-20693-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 6020449 INFO  (jetty-launcher-20683-thread-2) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 6020449 INFO  (zkConnectionManagerCallback-20691-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 6020449 INFO  (jetty-launcher-20683-thread-4) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 6020551 INFO  (jetty-launcher-20683-thread-5) [     ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 6020551 INFO  (jetty-launcher-20683-thread-4) [     ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 6020551 INFO  (jetty-launcher-20683-thread-2) [     ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 6020551 INFO  (jetty-launcher-20683-thread-3) [     ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 6020723 INFO  (jetty-launcher-20683-thread-1) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
   [junit4]   2> 6020723 WARN  (jetty-launcher-20683-thread-1) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@67a224e[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 6020723 WARN  (jetty-launcher-20683-thread-1) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@67a224e[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 6020725 WARN  (jetty-launcher-20683-thread-1) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@357c5d08[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 6020726 WARN  (jetty-launcher-20683-thread-1) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@357c5d08[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 6020727 INFO  (jetty-launcher-20683-thread-1) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45263/solr
   [junit4]   2> 6020727 INFO  (jetty-launcher-20683-thread-1) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 6020728 INFO  (zkConnectionManagerCallback-20701-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 6020728 INFO  (jetty-launcher-20683-thread-1) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 6020879 INFO  (jetty-launcher-20683-thread-5) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
   [junit4]   2> 6020880 WARN  (jetty-launcher-20683-thread-5) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@3722c986[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 6020880 WARN  (jetty-launcher-20683-thread-5) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@3722c986[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 6020882 WARN  (jetty-launcher-20683-thread-5) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@2d0be825[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 6020882 WARN  (jetty-launcher-20683-thread-5) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@2d0be825[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 6020883 INFO  (jetty-launcher-20683-thread-5) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45263/solr
   [junit4]   2> 6020884 INFO  (jetty-launcher-20683-thread-5) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 6020885 INFO  (zkConnectionManagerCallback-20709-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 6020885 INFO  (jetty-launcher-20683-thread-5) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 6020959 INFO  (jetty-launcher-20683-thread-3) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
   [junit4]   2> 6020960 WARN  (jetty-launcher-20683-thread-3) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@75ae5f2c[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 6020960 WARN  (jetty-launcher-20683-thread-3) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@75ae5f2c[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 6020964 WARN  (jetty-launcher-20683-thread-3) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@47e3e18f[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 6020964 WARN  (jetty-launcher-20683-thread-3) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@47e3e18f[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 6020965 INFO  (jetty-launcher-20683-thread-3) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45263/solr
   [junit4]   2> 6020966 INFO  (jetty-launcher-20683-thread-3) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 6020967 INFO  (zkConnectionManagerCallback-20716-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 6020967 INFO  (jetty-launcher-20683-thread-3) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 6020996 INFO  (jetty-launcher-20683-thread-2) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
   [junit4]   2> 6020996 WARN  (jetty-launcher-20683-thread-2) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@2dc7acf7[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 6020996 WARN  (jetty-launcher-20683-thread-2) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@2dc7acf7[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 6020998 WARN  (jetty-launcher-20683-thread-2) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@4add62c4[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 6020998 WARN  (jetty-launcher-20683-thread-2) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@4add62c4[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 6021000 INFO  (jetty-launcher-20683-thread-2) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45263/solr
   [junit4]   2> 6021001 INFO  (jetty-launcher-20683-thread-2) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 6021001 INFO  (zkConnectionManagerCallback-20722-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 6021001 INFO  (jetty-launcher-20683-thread-2) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 6021057 INFO  (jetty-launcher-20683-thread-4) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
   [junit4]   2> 6021058 WARN  (jetty-launcher-20683-thread-4) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@5819a0a[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 6021058 WARN  (jetty-launcher-20683-thread-4) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@5819a0a[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 6021061 WARN  (jetty-launcher-20683-thread-4) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@6d59b9cb[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 6021061 WARN  (jetty-launcher-20683-thread-4) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@6d59b9cb[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 6021062 INFO  (jetty-launcher-20683-thread-4) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45263/solr
   [junit4]   2> 6021063 INFO  (jetty-launcher-20683-thread-1) [n:127.0.0.1:42540_solr     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 6021063 INFO  (jetty-launcher-20683-thread-4) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 6021063 INFO  (zkConnectionManagerCallback-20728-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 6021063 INFO  (jetty-launcher-20683-thread-1) [n:127.0.0.1:42540_solr     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 6021064 INFO  (zkConnectionManagerCallback-20730-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 6021064 INFO  (jetty-launcher-20683-thread-4) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 6021163 INFO  (jetty-launcher-20683-thread-5) [n:127.0.0.1:45317_solr     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 6021164 INFO  (zkConnectionManagerCallback-20732-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 6021164 INFO  (jetty-launcher-20683-thread-5) [n:127.0.0.1:45317_solr     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 6021264 INFO  (jetty-launcher-20683-thread-2) [n:127.0.0.1:37817_solr     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 6021265 INFO  (zkConnectionManagerCallback-20734-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 6021265 INFO  (jetty-launcher-20683-thread-2) [n:127.0.0.1:37817_solr     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 6021366 INFO  (jetty-launcher-20683-thread-3) [n:127.0.0.1:39700_solr     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 6021366 INFO  (jetty-launcher-20683-thread-4) [n:127.0.0.1:33174_solr     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 6021367 INFO  (zkConnectionManagerCallback-20736-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 6021367 INFO  (jetty-launcher-20683-thread-3) [n:127.0.0.1:39700_solr     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 6021368 INFO  (zkConnectionManagerCallback-20738-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 6021368 INFO  (jetty-launcher-20683-thread-4) [n:127.0.0.1:33174_solr     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 6021469 INFO  (jetty-launcher-20683-thread-5) [n:127.0.0.1:45317_solr     ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:45317_solr
   [junit4]   2> 6021471 INFO  (jetty-launcher-20683-thread-5) [n:127.0.0.1:45317_solr     ] o.a.s.c.Overseer Overseer (id=75890327897702414-127.0.0.1:45317_solr-n_0000000000) starting
   [junit4]   2> 6021478 INFO  (OverseerStateUpdate-75890327897702414-127.0.0.1:45317_solr-n_0000000000) [n:127.0.0.1:45317_solr     ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:45317_solr
   [junit4]   2> 6021478 INFO  (jetty-launcher-20683-thread-5) [n:127.0.0.1:45317_solr     ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45317_solr
   [junit4]   2> 6021479 DEBUG (OverseerAutoScalingTriggerThread-75890327897702414-127.0.0.1:45317_solr-n_0000000000) [     ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
   [junit4]   2> 6021480 INFO  (OverseerStateUpdate-75890327897702414-127.0.0.1:45317_solr-n_0000000000) [n:127.0.0.1:45317_solr     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 6021499 DEBUG (OverseerAutoScalingTriggerThread-75890327897702414-127.0.0.1:45317_solr-n_0000000000) [     ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
   [junit4]   2> 6021499 WARN  (jetty-launcher-20683-thread-5) [n:127.0.0.1:45317_solr     ] o.a.s.c.CoreContainer Not all security plugins configured!  authentication=disabled authorization=disabled.  Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external.  See https://s.apache.org/solrsecurity for more info
   [junit4]   2> 6021499 DEBUG (OverseerAutoScalingTriggerThread-75890327897702414-127.0.0.1:45317_solr-n_0000000000) [     ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 0
   [junit4]   2> 6021499 DEBUG (OverseerAutoScalingTriggerThread-75890327897702414-127.0.0.1:45317_solr-n_0000000000) [     ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
   [junit4]   2> 6021520 INFO  (jetty-launcher-20683-thread-5) [n:127.0.0.1:45317_solr     ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 6021527 DEBUG (OverseerAutoScalingTriggerThread-75890327897702414-127.0.0.1:45317_solr-n_0000000000) [     ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:45317_solr]
   [junit4]   2> 6021531 INFO  (jetty-launcher-20683-thread-2) [n:127.0.0.1:37817_solr     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 6021531 INFO  (jetty-launcher-20683-thread-1) [n:127.0.0.1:42540_solr     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 6021537 DEBUG (ScheduledTrigger-15440-thread-1) [     ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
   [junit4]   2> 6021537 INFO  (jetty-launcher-20683-thread-4) [n:127.0.0.1:33174_solr     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 6021539 INFO  (jetty-launcher-20683-thread-1) [n:127.0.0.1:42540_solr     ] o.a.s.c.ZkController Publish node=127.0.0.1:42540_solr as DOWN
   [junit4]   2> 6021539 INFO  (jetty-launcher-20683-thread-3) [n:127.0.0.1:39700_solr     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 6021539 INFO  (jetty-launcher-20683-thread-2) [n:127.0.0.1:37817_solr     ] o.a.s.c.ZkController Publish node=127.0.0.1:37817_solr as DOWN
   [junit4]   2> 6021540 INFO  (jetty-launcher-20683-thread-1) [n:127.0.0.1:42540_solr     ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 6021540 INFO  (jetty-launcher-20683-thread-1) [n:127.0.0.1:42540_solr     ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:42540_solr
   [junit4]   2> 6021541 INFO  (jetty-launcher-20683-thread-2) [n:127.0.0.1:37817_solr     ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 6021541 INFO  (jetty-launcher-20683-thread-2) [n:127.0.0.1:37817_solr     ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37817_solr
   [junit4]   2> 6021542 WARN  (jetty-launcher-20683-thread-1) [n:127.0.0.1:42540_solr     ] o.a.s.c.CoreContainer Not all security plugins configured!  authentication=disabled authorization=disabled.  Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external.  See https://s.apache.org/solrsecurity for more info
   [junit4]   2> 6021547 INFO  (jetty-launcher-20683-thread-5) [n:127.0.0.1:45317_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45317.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@25dcf6f2
   [junit4]   2> 6021555 DEBUG (OverseerAutoScalingTriggerThread-75890327897702414-127.0.0.1:45317_solr-n_0000000000) [     ] o.a.s.c.a.OverseerTriggerThread -- deactivating old nodeLost / nodeAdded markers
   [junit4]   2> 6021555 INFO  (jetty-launcher-20683-thread-5) [n:127.0.0.1:45317_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45317.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@25dcf6f2
   [junit4]   2> 6021555 INFO  (jetty-launcher-20683-thread-5) [n:127.0.0.1:45317_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45317.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@25dcf6f2
   [junit4]   2> 6021555 INFO  (jetty-launcher-20683-thread-4) [n:127.0.0.1:33174_solr     ] o.a.s.c.ZkController Publish node=127.0.0.1:33174_solr as DOWN
   [junit4]   2> 6021556 WARN  (jetty-launcher-20683-thread-2) [n:127.0.0.1:37817_solr     ] o.a.s.c.CoreContainer Not all security plugins configured!  authentication=disabled authorization=disabled.  Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external.  See https://s.apache.org/solrsecurity for more info
   [junit4]   2> 6021559 INFO  (zkCallback-20737-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (3)
   [junit4]   2> 6021559 INFO  (zkCallback-20735-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (3)
   [junit4]   2> 6021559 INFO  (zkCallback-20731-thread-2) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (3)
   [junit4]   2> 6021560 INFO  (jetty-launcher-20683-thread-4) [n:127.0.0.1:33174_solr     ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 6021560 INFO  (jetty-launcher-20683-thread-4) [n:127.0.0.1:33174_solr     ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33174_solr
   [junit4]   2> 6021560 INFO  (jetty-launcher-20683-thread-5) [n:127.0.0.1:45317_solr     ] o.a.s.p.PackageLoader /packages.json updated to version -1
   [junit4]   2> 6021561 INFO  (jetty-launcher-20683-thread-5) [n:127.0.0.1:45317_solr     ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.SearchRateTriggerIntegrationTest_9BD8837E45B27770-001/tempDir-001/node4/.
   [junit4]   2> 6021562 INFO  (jetty-launcher-20683-thread-3) [n:127.0.0.1:39700_solr     ] o.a.s.c.ZkController Publish node=127.0.0.1:39700_solr as DOWN
   [junit4]   2> 6021562 INFO  (zkCallback-20731-thread-2) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 6021562 INFO  (jetty-launcher-20683-thread-1) [n:127.0.0.1:42540_solr     ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 6021563 INFO  (zkCallback-20727-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (3)
   [junit4]   2> 6021567 INFO  (zkCallback-20737-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 6021567 WARN  (jetty-launcher-20683-thread-4) [n:127.0.0.1:33174_solr     ] o.a.s.c.CoreContainer Not all security plugins configured!  authentication=disabled authorization=disabled.  Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external.  See https://s.apache.org/solrsecurity for more info
   [junit4]   2> 6021571 INFO  (zkCallback-20735-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 6021571 DEBUG (OverseerAutoScalingTriggerThread-75890327897702414-127.0.0.1:45317_solr-n_0000000000) [     ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
   [junit4]   2> 6021577 INFO  (zkCallback-20727-thread-2) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 6021583 INFO  (jetty-launcher-20683-thread-3) [n:127.0.0.1:39700_solr     ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 6021583 INFO  (jetty-launcher-20683-thread-3) [n:127.0.0.1:39700_solr     ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:39700_solr
   [junit4]   2> 6021584 INFO  (zkCallback-20737-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 6021585 INFO  (zkCallback-20731-thread-2) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 6021585 INFO  (zkCallback-20735-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 6021585 INFO  (zkCallback-20733-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (4)
   [junit4]   2> 6021585 WARN  (jetty-launcher-20683-thread-3) [n:127.0.0.1:39700_solr     ] o.a.s.c.CoreContainer Not all security plugins configured!  authentication=disabled authorization=disabled.  Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external.  See https://s.apache.org/solrsecurity for more info
   [junit4]   2> 6021596 INFO  (zkCallback-20727-thread-2) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 6021599 INFO  (zkCallback-20733-thread-2) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 6021599 INFO  (jetty-launcher-20683-thread-1) [n:127.0.0.1:42540_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_42540.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@25dcf6f2
   [junit4]   2> 6021608 INFO  (jetty-launcher-20683-thread-1) [n:127.0.0.1:42540_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_42540.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@25dcf6f2
   [junit4]   2> 6021608 INFO  (jetty-launcher-20683-thread-1) [n:127.0.0.1:42540_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_42540.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@25dcf6f2
   [junit4]   2> 6021609 INFO  (jetty-launcher-20683-thread-1) [n:127.0.0.1:42540_solr     ] o.a.s.p.PackageLoader /packages.json updated to version -1
   [junit4]   2> 6021610 INFO  (jetty-launcher-20683-thread-1) [n:127.0.0.1:42540_solr     ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.SearchRateTriggerIntegrationTest_9BD8837E45B27770-001/tempDir-001/node1/.
   [junit4]   2> 6021611 INFO  (jetty-launcher-20683-thread-2) [n:127.0.0.1:37817_solr     ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 6021619 INFO  (jetty-launcher-20683-thread-4) [n:127.0.0.1:33174_solr     ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 6021619 INFO  (jetty-launcher-20683-thread-3) [n:127.0.0.1:39700_solr     ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 6021635 INFO  (jetty-launcher-20683-thread-2) [n:127.0.0.1:37817_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_37817.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@25dcf6f2
   [junit4]   2> 6021650 INFO  (jetty-launcher-20683-thread-3) [n:127.0.0.1:39700_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_39700.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@25dcf6f2
   [junit4]   2> 6021668 INFO  (jetty-launcher-20683-thread-2) [n:127.0.0.1:37817_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_37817.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@25dcf6f2
   [junit4]   2> 6021668 INFO  (jetty-launcher-20683-thread-2) [n:127.0.0.1:37817_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_37817.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@25dcf6f2
   [junit4]   2> 6021671 INFO  (jetty-launcher-20683-thread-2) [n:127.0.0.1:37817_solr     ] o.a.s.p.PackageLoader /packages.json updated to version -1
   [junit4]   2> 6021672 INFO  (jetty-launcher-20683-thread-2) [n:127.0.0.1:37817_solr     ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.SearchRateTriggerIntegrationTest_9BD8837E45B27770-001/tempDir-001/node2/.
   [junit4]   2> 6021676 INFO  (jetty-launcher-20683-thread-3) [n:127.0.0.1:39700_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_39700.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@25dcf6f2
   [junit4]   2> 6021676 INFO  (jetty-launcher-20683-thread-3) [n:127.0.0.1:39700_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_39700.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@25dcf6f2
   [junit4]   2> 6021678 INFO  (jetty-launcher-20683-thread-3) [n:127.0.0.1:39700_solr     ] o.a.s.p.PackageLoader /packages.json updated to version -1
   [junit4]   2> 6021678 INFO  (jetty-launcher-20683-thread-3) [n:127.0.0.1:39700_solr     ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.SearchRateTriggerIntegrationTest_9BD8837E45B27770-001/tempDir-001/node3/.
   [junit4]   2> 6021711 INFO  (jetty-launcher-20683-thread-4) [n:127.0.0.1:33174_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33174.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@25dcf6f2
   [junit4]   2> 6021720 INFO  (jetty-launcher-20683-thread-4) [n:127.0.0.1:33174_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33174.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@25dcf6f2
   [junit4]   2> 6021720 INFO  (jetty-launcher-20683-thread-4) [n:127.0.0.1:33174_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33174.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@25dcf6f2
   [junit4]   2> 6021722 INFO  (jetty-launcher-20683-thread-4) [n:127.0.0.1:33174_solr     ] o.a.s.p.PackageLoader /packages.json updated to version -1
   [junit4]   2> 6021723 INFO  (jetty-launcher-20683-thread-4) [n:127.0.0.1:33174_solr     ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.SearchRateTriggerIntegrationTest_9BD8837E45B27770-001/tempDir-001/node5/.
   [junit4]   2> 6021840 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[9BD8837E45B27770]-worker) [     ] o.a.s.c.MiniSolrCloudCluster waitForAllNodes: numServers=5
   [junit4]   2> 6021841 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[9BD8837E45B27770]-worker) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 6021842 INFO  (zkConnectionManagerCallback-20763-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 6021842 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[9BD8837E45B27770]-worker) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 6021843 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[9BD8837E45B27770]-worker) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (5)
   [junit4]   2> 6021844 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[9BD8837E45B27770]-worker) [     ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45263/solr ready
   [junit4]   2> 6021848 INFO  (qtp440332187-67931) [n:127.0.0.1:37817_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 6021851 INFO  (qtp1753290285-67932) [n:127.0.0.1:33174_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 6021852 INFO  (qtp440332187-67940) [n:127.0.0.1:37817_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 6021853 INFO  (qtp1010252685-67925) [n:127.0.0.1:39700_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 6021855 INFO  (qtp121092900-67904) [n:127.0.0.1:42540_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 6021856 INFO  (qtp716192567-67935) [n:127.0.0.1:45317_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 6021856 DEBUG (qtp1753290285-67922) [n:127.0.0.1:33174_solr     ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 6021857 INFO  (qtp1753290285-67922) [n:127.0.0.1:33174_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/autoscaling params={wt=javabin&version=2} status=0 QTime=7
   [junit4]   2> 6021857 DEBUG (zkCallback-20731-thread-2) [     ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2
   [junit4]   2> 6021858 DEBUG (OverseerAutoScalingTriggerThread-75890327897702414-127.0.0.1:45317_solr-n_0000000000) [     ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
   [junit4]   2> 6021865 DEBUG (OverseerAutoScalingTriggerThread-75890327897702414-127.0.0.1:45317_solr-n_0000000000) [     ] o.a.s.c.a.OverseerTriggerThread -- deactivating old nodeLost / nodeAdded markers
   [junit4]   2> 6021869 DEBUG (OverseerAutoScalingTriggerThread-75890327897702414-127.0.0.1:45317_solr-n_0000000000) [     ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
   [junit4]   2> 6021933 INFO  (TEST-SearchRateTriggerIntegrationTest.testAboveSearchRate-seed#[9BD8837E45B27770]) [     ] o.a.s.SolrTestCaseJ4 ###Starting testAboveSearchRate
   [junit4]   2> 6021934 INFO  (TEST-SearchRateTriggerIntegrationTest.testAboveSearchRate-seed#[9BD8837E45B27770]) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (5)
   [junit4]   2> 6021940 INFO  (TEST-SearchRateTriggerIntegrationTest.testAboveSearchRate-seed#[9BD8837E45B27770]) [     ] o.a.s.c.a.SearchRateTriggerIntegrationTest /autoscaling.json reset, new znode version 3
   [junit4]   2> 6021940 DEBUG (zkCallback-20731-thread-2) [     ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 3
   [junit4]   2> 6021940 DEBUG (OverseerAutoScalingTriggerThread-75890327897702414-127.0.0.1:45317_solr-n_0000000000) [     ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
   [junit4]   2> 6021944 DEBUG (OverseerAutoScalingTriggerThread-75890327897702414-127.0.0.1:45317_solr-n_0000000000) [     ] o.a.s.c.a.OverseerTriggerThread -- deactivating old nodeLost / nodeAdded markers
   [junit4]   2> 6021945 DEBUG (OverseerAutoScalingTriggerThread-75890327897702414-127.0.0.1:45317_solr-n_0000000000) [     ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
   [junit4]   2> 6021946 INFO  (qtp1753290285-67918) [n:127.0.0.1:33174_solr     ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf&name=aboveRate_collection&nrtReplicas=2&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 6021948 INFO  (OverseerThreadFactory-15442-thread-1-processing-n:127.0.0.1:45317_solr) [n:127.0.0.1:45317_solr     ] o.a.s.c.a.c.CreateCollectionCmd Create collection aboveRate_collection
   [junit4]   2> 6022056 INFO  (OverseerStateUpdate-75890327897702414-127.0.0.1:45317_solr-n_0000000000) [n:127.0.0.1:45317_solr     ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"aboveRate_collection",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"aboveRate_collection_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:37817/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 6022058 INFO  (OverseerStateUpdate-75890327897702414-127.0.0.1:45317_solr-n_0000000000) [n:127.0.0.1:45317_solr     ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"aboveRate_collection",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"aboveRate_collection_shard1_replica_n2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:45317/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 6022261 INFO  (qtp440332187-67916) [n:127.0.0.1:37817_solr    x:aboveRate_collection_shard1_replica_n1 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=aboveRate_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=aboveRate_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 6022261 INFO  (qtp716192567-67939) [n:127.0.0.1:45317_solr    x:aboveRate_collection_shard1_replica_n2 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=aboveRate_collection_shard1_replica_n2&action=CREATE&numShards=1&collection=aboveRate_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 6022261 INFO  (qtp716192567-67939) [n:127.0.0.1:45317_solr    x:aboveRate_collection_shard1_replica_n2 ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 6023276 INFO  (qtp716192567-67939) [n:127.0.0.1:45317_solr c:aboveRate_collection s:shard1 r:core_node4 x:aboveRate_collection_shard1_replica_n2 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.4.0
   [junit4]   2> 6023276 INFO  (qtp440332187-67916) [n:127.0.0.1:37817_solr c:aboveRate_collection s:shard1 r:core_node3 x:aboveRate_collection_shard1_replica_n1 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.4.0
   [junit4]   2> 6023300 INFO  (qtp716192567-67939) [n:127.0.0.1:45317_solr c:aboveRate_collection s:shard1 r:core_node4 x:aboveRate_collection_shard1_replica_n2 ] o.a.s.s.IndexSchema [aboveRate_collection_shard1_replica_n2] Schema name=minimal
   [junit4]   2> 6023300 INFO  (qtp440332187-67916) [n:127.0.0.1:37817_solr c:aboveRate_collection s:shard1 r:core_node3 x:aboveRate_collection_shard1_replica_n1 ] o.a.s.s.IndexSchema [aboveRate_collection_shard1_replica_n1] Schema name=minimal
   [junit4]   2> 6023303 INFO  (qtp716192567-67939) [n:127.0.0.1:45317_solr c:aboveRate_collection s:shard1 r:core_node4 x:aboveRate_collection_shard1_replica_n2 ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 6023303 INFO  (qtp440332187-67916) [n:127.0.0.1:37817_solr c:aboveRate_collection s:shard1 r:core_node3 x:aboveRate_collection_shard1_replica_n1 ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 6023303 INFO  (qtp716192567-67939) [n:127.0.0.1:45317_solr c:aboveRate_collection s:shard1 r:core_node4 x:aboveRate_collection_shard1_replica_n2 ] o.a.s.c.CoreContainer Creating SolrCore 'aboveRate_collection_shard1_replica_n2' using configuration from collection aboveRate_collection, trusted=true
   [junit4]   2> 6023303 INFO  (qtp440332187-67916) [n:127.0.0.1:37817_solr c:aboveRate_collection s:shard1 r:core_node3 x:aboveRate_collection_shard1_replica_n1 ] o.a.s.c.CoreContainer Creating SolrCore 'aboveRate_collection_shard1_replica_n1' using configuration from collection aboveRate_collection, trusted=true
   [junit4]   2> 6023304 INFO  (qtp716192567-67939) [n:127.0.0.1:45317_solr c:aboveRate_collection s:shard1 r:core_node4 x:aboveRate_collection_shard1_replica_n2 ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45317.solr.core.aboveRate_collection.shard1.replica_n2' (registry 'solr.core.aboveRate_collection.shard1.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@25dcf6f2
   [junit4]   2> 6023304 INFO  (qtp440332187-67916) [n:127.0.0.1:37817_solr c:aboveRate_collection s:shard1 r:core_node3 x:aboveRate_collection_shard1_replica_n1 ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_37817.solr.core.aboveRate_collection.shard1.replica_n1' (registry 'solr.core.aboveRate_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@25dcf6f2
   [junit4]   2> 6023304 INFO  (qtp440332187-67916) [n:127.0.0.1:37817_solr c:aboveRate_collection s:shard1 r:core_node3 x:aboveRate_collection_shard1_replica_n1 ] o.a.s.c.SolrCore [[aboveRate_collection_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.SearchRateTriggerIntegrationTest_9BD8837E45B27770-001/tempDir-001/node2/aboveRate_collection_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.SearchRateTriggerIntegrationTest_9BD8837E45B27770-001/tempDir-001/node2/./aboveRate_collection_shard1_replica_n1/data/]
   [junit4]   2> 6023304 INFO  (qtp716192567-67939) [n:127.0.0.1:45317_solr c:aboveRate_collection s:shard1 r:core_node4 x:aboveRate_collection_shard1_replica_n2 ] o.a.s.c.SolrCore [[aboveRate_collection_shard1_replica_n2] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.SearchRateTriggerIntegrationTest_9BD8837E45B27770-001/tempDir-001/node4/aboveRate_collection_shard1_replica_n2], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.SearchRateTriggerIntegrationTest_9BD8837E45B27770-001/tempDir-001/node4/./aboveRate_collection_shard1_replica_n2/data/]
   [junit4]   2> 6023362 INFO  (qtp716192567-67939) [n:127.0.0.1:45317_solr c:aboveRate_collection s:shard1 r:core_node4 x:aboveRate_collection_shard1_replica_n2 ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 6023362 INFO  (qtp716192567-67939) [n:127.0.0.1:45317_solr c:aboveRate_collection s:shard1 r:core_node4 x:aboveRate_collection_shard1_replica_n2 ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 6023364 INFO  (qtp716192567-67939) [n:127.0.0.1:45317_solr c:aboveRate_collection s:shard1 r:core_node4 x:aboveRate_collection_shard1_replica_n2 ] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 6023364 INFO  (qtp716192567-67939) [n:127.0.0.1:45317_solr c:aboveRate_collection s:shard1 r:core_node4 x:aboveRate_collection_shard1_replica_n2 ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 6023365 INFO  (qtp716192567-67939) [n:127.0.0.1:45317_solr c:aboveRate_collection s:shard1 r:core_node4 x:aboveRate_collection_shard1_replica_n2 ] o.a.s.s.SolrIndexSearcher Opening [Searcher@1f0654d[aboveRate_collection_shard1_replica_n2] main]
   [junit4]   2> 6023365 INFO  (qtp440332187-67916) [n:127.0.0.1:37817_solr c:aboveRate_collection s:shard1 r:core_node3 x:aboveRate_collection_shard1_replica_n1 ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 6023365 INFO  (qtp440332187-67916) [n:127.0.0.1:37817_solr c:aboveRate_collection s:shard1 r:core_node3 x:aboveRate_collection_shard1_replica_n1 ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 6023366 INFO  (qtp716192567-67939) [n:127.0.0.1:45317_solr c:aboveRate_collection s:shard1 r:core_node4 x:aboveRate_collection_shard1_replica_n2 ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 6023366 INFO  (qtp440332187-67916) [n:127.0.0.1:37817_solr c:aboveRate_collection s:shard1 r:core_node3 x:aboveRate_collection_shard1_replica_n1 ] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 6023366 INFO  (qtp440332187-67916) [n:127.0.0.1:37817_solr c:aboveRate_collection s:shard1 r:core_node3 x:aboveRate_collection_shard1_replica_n1 ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 6023366 INFO  (qtp716192567-67939) [n:127.0.0.1:45317_solr c:aboveRate_collection s:shard1 r:core_node4 x:aboveRate_collection_shard1_replica_n2 ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 6023367 INFO  (qtp716192567-67939) [n:127.0.0.1:45317_solr c:aboveRate_collection s:shard1 r:core_node4 x:aboveRate_collection_shard1_replica_n2 ] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 6023367 INFO  (qtp716192567-67939) [n:127.0.0.1:45317_solr c:aboveRate_collection s:shard1 r:core_node4 x:aboveRate_collection_shard1_replica_n2 ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1652708322541830144
   [junit4]   2> 6023367 INFO  (qtp440332187-67916) [n:127.0.0.1:37817_solr c:aboveRate_collection s:shard1 r:core_node3 x:aboveRate_collection_shard1_replica_n1 ] o.a.s.s.SolrIndexSearcher Opening [Searcher@5543a53d[aboveRate_collection_shard1_replica_n1] main]
   [junit4]   2> 6023370 INFO  (searcherExecutor-15463-thread-1-processing-n:127.0.0.1:45317_solr x:aboveRate_collection_shard1_replica_n2 c:aboveRate_collection s:shard1 r:core_node4) [n:127.0.0.1:45317_solr c:aboveRate_collection s:shard1 r:core_node4 x:aboveRate_collection_shard1_replica_n2 ] o.a.s.c.SolrCore [aboveRate_collection_shard1_replica_n2] Registered new searcher Searcher@1f0654d[aboveRate_collection_shard1_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 6023372 INFO  (qtp440332187-67916) [n:127.0.0.1:37817_solr c:aboveRate_collection s:shard1 r:core_node3 x:aboveRate_collection_shard1_replica_n1 ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 6023372 INFO  (qtp440332187-67916) [n:127.0.0.1:37817_solr c:aboveRate_collection s:shard1 r:core_node3 x:aboveRate_collection_shard1_replica_n1 ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 6023372 INFO  (qtp440332187-67916) [n:127.0.0.1:37817_solr c:aboveRate_collection s:shard1 r:core_node3 x:aboveRate_collection_shard1_replica_n1 ] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 6023372 INFO  (qtp716192567-67939) [n:127.0.0.1:45317_solr c:aboveRate_collection s:shard1 r:core_node4 x:aboveRate_collection_shard1_replica_n2 ] o.a.s.c.ZkShardTerms Successful update of terms at /collections/aboveRate_collection/terms/shard1 to Terms{values={core_node4=0}, version=0}
   [junit4]   2> 6023372 INFO  (qtp716192567-67939) [n:127.0.0.1:45317_solr c:aboveRate_collection s:shard1 r:core_node4 x:aboveRate_collection_shard1_replica_n2 ] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/aboveRate_collection/leaders/shard1
   [junit4]   2> 6023372 INFO  (qtp440332187-67916) [n:127.0.0.1:37817_solr c:aboveRate_collection s:shard1 r:core_node3 x:aboveRate_collection_shard1_replica_n1 ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1652708322547073024
   [junit4]   2> 6023375 INFO  (searcherExecutor-15464-thread-1-processing-n:127.0.0.1:37817_solr x:aboveRate_collection_shard1_replica_n1 c:aboveRate_collection s:shard1 r:core_node3) [n:127.0.0.1:37817_solr c:aboveRate_collection s:shard1 r:core_node3 x:aboveRate_collection_shard1_replica_n1 ] o.a.s.c.SolrCore [aboveRate_collection_shard1_replica_n1] Registered new searcher Searcher@5543a53d[aboveRate_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 6023377 INFO  (qtp716192567-67939) [n:127.0.0.1:45317_solr c:aboveRate_collection s:shard1 r:core_node4 x:aboveRate_collection_shard1_replica_n2 ] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard1: total=2 found=1 timeoutin=9999ms
   [junit4]   2> 6023378 INFO  (qtp440332187-67916) [n:127.0.0.1:37817_solr c:aboveRate_collection s:shard1 r:core_node3 x:aboveRate_collection_shard1_replica_n1 ] o.a.s.c.ZkShardTerms Successful update of terms at /collections/aboveRate_collection/terms/shard1 to Terms{values={core_node3=0, core_node4=0}, version=1}
   [junit4]   2> 6023378 INFO  (qtp440332187-67916) [n:127.0.0.1:37817_solr c:aboveRate_collection s:shard1 r:core_node3 x:aboveRate_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/aboveRate_collection/leaders/shard1
   [junit4]   2> 6023878 INFO  (qtp716192567-67939) [n:127.0.0.1:45317_solr c:aboveRate_collection s:shard1 r:core_node4 x:aboveRate_collection_shard1_replica_n2 ] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 6023878 INFO  (qtp716192567-67939) [n:127.0.0.1:45317_solr c:aboveRate_collection s:shard1 r:core_node4 x:aboveRate_collection_shard1_replica_n2 ] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 6023878 INFO  (qtp716192567-67939) [n:127.0.0.1:45317_solr c:aboveRate_collection s:shard1 r:core_node4 x:aboveRate_collection_shard1_replica_n2 ] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:45317/solr/aboveRate_collection_shard1_replica_n2/
   [junit4]   2> 6023879 INFO  (qtp716192567-67939) [n:127.0.0.1:45317_solr c:aboveRate_collection s:shard1 r:core_node4 x:aboveRate_collection_shard1_replica_n2 ] o.a.s.u.PeerSync PeerSync: core=aboveRate_collection_shard1_replica_n2 url=http://127.0.0.1:45317/solr START replicas=[http://127.0.0.1:37817/solr/aboveRate_collection_shard1_replica_n1/] nUpdates=100
   [junit4]   2> 6023879 INFO  (qtp716192567-67939) [n:127.0.0.1:45317_solr c:aboveRate_collection s:shard1 r:core_node4 x:aboveRate_collection_shard1_replica_n2 ] o.a.s.u.PeerSync PeerSync: core=aboveRate_collection_shard1_replica_n2 url=http://127.0.0.1:45317/solr DONE.  We have no versions.  sync failed.
   [junit4]   2> 6023882 INFO  (qtp440332187-67931) [n:127.0.0.1:37817_solr c:aboveRate_collection s:shard1 r:core_node3 x:aboveRate_collection_shard1_replica_n1 ] o.a.s.c.S.Request [aboveRate_collection_shard1_replica_n1]  webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=1
   [junit4]   2> 6023882 INFO  (qtp716192567-67939) [n:127.0.0.1:45317_solr c:aboveRate_collection s:shard1 r:core_node4 x:aboveRate_collection_shard1_replica_n2 ] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
   [junit4]   2> 6023882 INFO  (qtp716192567-67939) [n:127.0.0.1:45317_solr c:aboveRate_collection s:shard1 r:core_node4 x:aboveRate_collection_shard1_replica_n2 ] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 6023882 INFO  (qtp716192567-67939) [n:127.0.0.1:45317_solr c:aboveRate_collection s:shard1 r:core_node4 x:aboveRate_collection_shard1_replica_n2 ] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/aboveRate_collection/leaders/shard1/leader after winning as /collections/aboveRate_collection/leader_elect/shard1/election/75890327897702414-core_node4-n_0000000000
   [junit4]   2> 6023883 INFO  (qtp716192567-67939) [n:127.0.0.1:45317_solr c:aboveRate_collection s:shard1 r:core_node4 x:aboveRate_collection_shard1_replica_n2 ] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:45317/solr/aboveRate_collection_shard1_replica_n2/ shard1
   [junit4]   2> 6023985 INFO  (zkCallback-20731-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/aboveRate_collection/state.json] for collection [aboveRate_collection] has occurred - updating... (live nodes size: [5])
   [junit4]   2> 6023986 INFO  (qtp716192567-67939) [n:127.0.0.1:45317_solr c:aboveRate_collection s:shard1 r:core_node4 x:aboveRate_collection_shard1_replica_n2 ] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 6023989 INFO  (qtp716192567-67939) [n:127.0.0.1:45317_solr c:aboveRate_collection s:shard1 r:core_node4 x:aboveRate_collection_shard1_replica_n2 ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=aboveRate_collection_shard1_replica_n2&action=CREATE&numShards=1&collection=aboveRate_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1728
   [junit4]   2> 6024088 INFO  (zkCallback-20731-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/aboveRate_collection/state.json] for collection [aboveRate_collection] has occurred - updating... (live nodes size: [5])
   [junit4]   2> 6024088 INFO  (zkCallback-20731-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/aboveRate_collection/state.json] for collection [aboveRate_collection] has occurred - updating... (live nodes size: [5])
   [junit4]   2> 6024384 INFO  (qtp440332187-67916) [n:127.0.0.1:37817_solr c:aboveRate_collection s:shard1 r:core_node3 x:aboveRate_collection_shard1_replica_n1 ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=aboveRate_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=aboveRate_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2124
   [junit4]   2> 6024387 INFO  (qtp1753290285-67918) [n:127.0.0.1:33174_solr     ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
   [junit4]   2> 6024485 INFO  (zkCallback-20737-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/aboveRate_collection/state.json] for collection [aboveRate_collection] has occurred - updating... (live nodes size: [5])
   [junit4]   2> 6024485 INFO  (zkCallback-20733-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/aboveRate_collection/state.json] for collection [aboveRate_collection] has occurred - updating... (live nodes size: [5])
   [junit4]   2> 6024485 INFO  (zkCallback-20731-thread-3) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/aboveRate_collection/state.json] for collection [aboveRate_collection] has occurred - updating... (live nodes size: [5])
   [junit4]   2> 6024485 INFO  (zkCallback-20733-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/aboveRate_collection/state.json] for collection [aboveRate_collection] has occurred - updating... (live nodes size: [5])
   [junit4]   2> 6024485 INFO  (zkCallback-20731-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/aboveRate_collection/state.json] for collection [aboveRate_collection] has occurred - updating... (live nodes size: [5])
   [junit4]   2> 6024485 INFO  (qtp1753290285-67918) [n:127.0.0.1:33174_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf&name=aboveRate_collection&nrtReplicas=2&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=2539
   [junit4]   2> 6024599 INFO  (qtp1753290285-67932) [n:127.0.0.1:33174_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 6024602 INFO  (qtp121092900-67901) [n:127.0.0.1:42540_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 6024604 INFO  (qtp716192567-67927) [n:127.0.0.1:45317_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.aboveRate_collection.shard1.replica_n2:INDEX.sizeInBytes} status=0 QTime=1
   [junit4]   2> 6024605 INFO  (qtp716192567-67935) [n:127.0.0.1:45317_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 6024607 INFO  (qtp440332187-67940) [n:127.0.0.1:37817_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.aboveRate_collection.shard1.replica_n1:INDEX.sizeInBytes} status=0 QTime=0
   [junit4]   2> 6024608 INFO  (qtp440332187-67921) [n:127.0.0.1:37817_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 6024609 INFO  (qtp1010252685-67928) [n:127.0.0.1:39700_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 6024609 DEBUG (qtp121092900-67906) [n:127.0.0.1:42540_solr     ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 6024610 INFO  (qtp121092900-67906) [n:127.0.0.1:42540_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=0 QTime=22
   [junit4]   2> 6024611 DEBUG (zkCallback-20731-thread-1) [     ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 4
   [junit4]   2> 6024611 DEBUG (OverseerAutoScalingTriggerThread-75890327897702414-127.0.0.1:45317_solr-n_0000000000) [     ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 4
   [junit4]   2> 6024611 DEBUG (OverseerAutoScalingTriggerThread-75890327897702414-127.0.0.1:45317_solr-n_0000000000) [     ] o.a.s.c.a.OverseerTriggerThread -- deactivating old nodeLost / nodeAdded markers
   [junit4]   2> 6024612 DEBUG (OverseerAutoScalingTriggerThread-75890327897702414-127.0.0.1:45317_solr-n_0000000000) [     ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion 4
   [junit4]   2> 6024613 INFO  (qtp1753290285-67933) [n:127.0.0.1:33174_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 6024614 INFO  (qtp121092900-67904) [n:127.0.0.1:42540_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 6024614 INFO  (qtp716192567-67937) [n:127.0.0.1:45317_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.aboveRate_collection.shard1.replica_n2:INDEX.sizeInBytes} status=0 QTime=0
   [junit4]   2> 6024616 INFO  (qtp716192567-67939) [n:127.0.0.1:45317_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 6024616 INFO  (qtp440332187-67931) [n:127.0.0.1:37817_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.aboveRate_collection.shard1.replica_n1:INDEX.sizeInBytes} status=0 QTime=0
   [junit4]   2> 6024617 INFO  (qtp440332187-67916) [n:127.0.0.1:37817_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 6024618 INFO  (qtp1010252685-67930) [n:127.0.0.1:39700_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 6024619 DEBUG (qtp1753290285-67922) [n:127.0.0.1:33174_solr     ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 6024619 INFO  (qtp1753290285-67922) [n:127.0.0.1:33174_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/autoscaling params={wt=javabin&version=2} status=0 QTime=8
   [junit4]   2> 6024619 DEBUG (zkCallback-20731-thread-1) [     ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 5
   [junit4]   2> 6024619 DEBUG (OverseerAutoScalingTriggerThread-75890327897702414-127.0.0.1:45317_solr-n_0000000000) [     ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 5
   [junit4]   2> 6024620 DEBUG (OverseerAutoScalingTriggerThread-75890327897702414-127.0.0.1:45317_solr-n_0000000000) [     ] o.a.s.c.a.OverseerTriggerThread -- deactivating old nodeLost / nodeAdded markers
   [junit4]   2> 6024620 DEBUG (OverseerAutoScalingTriggerThread-75890327897702414-127.0.0.1:45317_solr-n_0000000000) [     ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 5, lastZnodeVersion 5
   [junit4]   2> 6024622 INFO  (qtp1753290285-67918) [n:127.0.0.1:33174_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 6024644 INFO  (qtp121092900-67905) [n:127.0.0.1:42540_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=20
   [junit4]   2> 6024645 INFO  (qtp716192567-67941) [n:127.0.0.1:45317_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.aboveRate_collection.shard1.replica_n2:INDEX.sizeInBytes} status=0 QTime=0
   [junit4]   2> 6024646 INFO  (qtp716192567-67927) [n:127.0.0.1:45317_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 6024647 INFO  (qtp440332187-67934) [n:127.0.0.1:37817_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.aboveRate_collection.shard1.replica_n1:INDEX.sizeInBytes} status=0 QTime=0
   [junit4]   2> 6024648 INFO  (qtp440332187-67940) [n:127.0.0.1:37817_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 6024649 INFO  (qtp1010252685-67924) [n:127.0.0.1:39700_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 6024649 DEBUG (qtp121092900-67903) [n:127.0.0.1:42540_solr     ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 6024649 INFO  (qtp121092900-67903) [n:127.0.0.1:42540_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=0 QTime=29
   [junit4]   2> 6024651 DEBUG (zkCallback-20731-thread-1) [     ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 6
   [junit4]   2> 6024651 DEBUG (OverseerAutoScalingTriggerThread-75890327897702414-127.0.0.1:45317_solr-n_0000000000) [     ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 6
   [junit4]   2> 6024651 DEBUG (OverseerAutoScalingTriggerThread-75890327897702414-127.0.0.1:45317_solr-n_0000000000) [     ] o.a.s.c.a.OverseerTriggerThread -- deactivating old nodeLost / nodeAdded markers
   [junit4]   2> 6024652 DEBUG (OverseerAutoScalingTriggerThread-75890327897702414-127.0.0.1:45317_solr-n_0000000000) [     ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion 6
   [junit4]   2> 6024652 INFO  (qtp1753290285-67929) [n:127.0.0.1:33174_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 6024653 INFO  (qtp121092900-67901) [n:127.0.0.1:42540_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 6024654 INFO  (qtp716192567-67939) [n:127.0.0.1:45317_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.aboveRate_collection.shard1.replica_n2:INDEX.sizeInBytes} status=0 QTime=0
   [junit4]   2> 6024655 INFO  (qtp716192567-67941) [n:127.0.0.1:45317_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 6024655 INFO  (qtp440332187-67921) [n:127.0.0.1:37817_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.aboveRate_collection.shard1.replica_n1:INDEX.sizeInBytes} status=0 QTime=0
   [junit4]   2> 6024656 INFO  (qtp440332187-67931) [n:127.0.0.1:37817_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 6024658 INFO  (qtp1010252685-67926) [n:127.0.0.1:39700_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 6024658 DEBUG (qtp716192567-67935) [n:127.0.0.1:45317_solr     ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 6024658 INFO  (qtp716192567-67935) [n:127.0.0.1:45317_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=0 QTime=7
   [junit4]   2> 6024658 DEBUG (zkCallback-20731-thread-1) [     ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 7
   [junit4]   2> 6024659 DEBUG (OverseerAutoScalingTriggerThread-75890327897702414-127.0.0.1:45317_solr-n_0000000000) [     ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 7
   [junit4]   2> 6024659 DEBUG (OverseerAutoScalingTriggerThread-75890327897702414-127.0.0.1:45317_solr-n_0000000000) [     ] o.a.s.c.a.OverseerTriggerThread -- deactivating old nodeLost / nodeAdded markers
   [junit4]   2> 6024659 DEBUG (OverseerAutoScalingTriggerThread-75890327897702414-127.0.0.1:45317_solr-n_0000000000) [     ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 7, lastZnodeVersion 7
   [junit4]   2> 6024659 INFO  (qtp716192567-67937) [n:127.0.0.1:45317_solr c:aboveRate_collection s:shard1 r:core_node4 x:aboveRate_collection_shard1_replica_n2 ] o.a.s.c.S.Request [aboveRate_collection_shard1_replica_n2]  webapp=/solr path=/select params={q=*:*&_stateVer_=aboveRate_collection:6&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 6024660 INFO  (qtp716192567-67939) [n:127.0.0.1:45317_solr c:aboveRate_collection s:shard1 r:core_node4 x:aboveRate_collection_shard1_replica_n2 ] o.a.s.c.S.Request [aboveRate_collection_shard1_replica_n2]  webapp=/solr path=/select params={q=*:*&_stateVer_=aboveRate_collection:6&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 6024661 INFO  (qtp440332187-67934) [n:127.0.0.1:37817_solr c:aboveRate_collection s:shard1 r:core_node3 x:aboveRate_collection_shard1_replica_n1 ] o.a.s.c.S.Request [aboveRate_collection_shard1_replica_n1]  webapp=/solr path=/select params={q=*:*&_stateVer_=aboveRate_collection:6&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 6024661 INFO  (qtp440332187-67940) [n:127.0.0.1:37817_solr c:aboveRate_collection s:shard1 r:core_node3 x:aboveRate_collection_shard1_replica_n1 ] o.a.s.c.S.Request [aboveRate_collection_shard1_replica_n1]  webapp=/solr path=/select params={q=*:*&_stateVer_=aboveRate_collection:6&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 6024662 INFO  (qtp440332187-67921) [n:127.0.0.1:37817_solr c:aboveRate_collection s:shard1 r:core_node3 x:aboveRate_collection_shard1_replica_n1 ] o.a.s.c.S.Request [aboveRate_collection_shard1_replica_n1]  webapp=/solr path=/select params={q=*:*&_stateVer_=aboveRate_collection:6&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 6024662 INFO  (qtp440332187-67931) [n:127.0.0.1:37817_solr c:aboveRate_collection s:shard1 r:core_node3 x:aboveRate_collection_shard1_replica_n1 ] o.a.s.c.S.Request [aboveRate_collection_shard1_replica_n1] 

[...truncated too long message...]

/solr/autoscaling.json
   [junit4]   2> 	15	/solr/aliases.json
   [junit4]   2> 	8	/solr/clusterstate.json
   [junit4]   2> 	3	/solr/collections/belowRate_collection/leader_elect/shard1/election/75890327897702417-core_node4-n_0000000000
   [junit4]   2> 	2	/solr/collections/aboveRate_collection/leader_elect/shard1/election/75890327897702414-core_node4-n_0000000000
   [junit4]   2> 	2	/solr/collections/aboveRate_collection/leader_elect/shard1/election/75890327897702416-core_node6-n_0000000002
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	94	/solr/overseer/queue
   [junit4]   2> 	36	/solr/overseer/collection-queue-work
   [junit4]   2> 	30	/solr/live_nodes
   [junit4]   2> 	27	/solr/collections
   [junit4]   2> 	6	/solr/autoscaling/events/search_rate_trigger2
   [junit4]   2> 	2	/solr/autoscaling/events/search_rate_trigger1
   [junit4]   2> 
   [junit4]   2> 6089718 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[9BD8837E45B27770]-worker) [     ] o.a.s.c.ZkTestServer waitForServerDown: 127.0.0.1:45263
   [junit4]   2> 6089719 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[9BD8837E45B27770]-worker) [     ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:45263
   [junit4]   2> 6089719 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[9BD8837E45B27770]-worker) [     ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 45263
   [junit4]   2> 6089719 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[9BD8837E45B27770]-worker) [     ] o.a.s.SolrTestCaseJ4 ------------------------------------------------------- Done waiting for tracked resources to be released
   [junit4]   2> Dec 12, 2019 10:12:26 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene84): {}, docValues:{}, maxPointsInLeafNode=1790, maxMBSortInHeap=7.0833402708048965, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@3f3fcedc), locale=ms-MY, timezone=Israel
   [junit4]   2> NOTE: Linux 4.4.0-112-generic amd64/Oracle Corporation 1.8.0_191 (64-bit)/cpus=4,threads=1,free=87600424,total=500695040
   [junit4]   2> NOTE: All tests run in this JVM: [TestRuleBasedAuthorizationPlugin, DistributedFacetPivotLargeTest, SpatialRPTFieldTypeTest, TolerantUpdateProcessorTest, TestTrie, AssignBackwardCompatibilityTest, TestLegacyField, TestOnReconnectListenerSupport, TestNonDefinedSimilarityFactory, TestReplicationHandlerDiskOverFlow, TestSolrDeletionPolicy2, TestManagedStopFilterFactory, ZkFailoverTest, HttpPartitionWithTlogReplicasTest, ConnectionManagerTest, TestOrdValues, TestOmitPositions, AnalyticsMergeStrategyTest, HdfsThreadLeakTest, NoCacheHeaderTest, TestLegacyTerms, ScriptEngineTest, SampleTest, TestRangeQuery, CdcrVersionReplicationTest, TestSolr4Spatial2, SystemCollectionCompatTest, ZkSolrClientTest, TestComponentsName, OpenExchangeRatesOrgProviderTest, TestRandomRequestDistribution, TestTolerantUpdateProcessorCloud, DeleteReplicaTest, TlogReplayBufferedWhileIndexingTest, TestNumericTerms32, DocValuesMultiTest, DataDrivenBlockJoinTest, JsonLoaderTest, DeleteInactiveReplicaTest, FullHLLTest, TestSchemaVersionResource, UniqFieldsUpdateProcessorFactoryTest, TestExclusionRuleCollectionAccess, TestCoreDiscovery, TestNamedUpdateProcessors, ResponseLogComponentTest, BlockJoinFacetSimpleTest, TestCharFilters, JavaBinAtomicUpdateMultivalueTest, BasicAuthIntegrationTest, SimpleMLTQParserTest, ChangedSchemaMergeTest, TestDistributedTracing, HdfsDirectoryFactoryTest, TestSolrConfigHandlerConcurrent, ConjunctionSolrSpellCheckerTest, SimplePostToolTest, TestInPlaceUpdatesStandalone, TestDistribIDF, UninvertDocValuesMergePolicyTest, AnalysisAfterCoreReloadTest, CollectionsAPISolrJTest, PreAnalyzedUpdateProcessorTest, TaggingAttributeTest, VMParamsZkACLAndCredentialsProvidersTest, BJQFilterAccessibleTest, TestExplainDocTransformer, TestHdfsCloudBackupRestore, TestFaceting, HdfsRecoveryZkTest, HighlighterConfigTest, CSVRequestHandlerTest, DistributedQueueTest, V2ApiIntegrationTest, HdfsAutoAddReplicasIntegrationTest, SpellPossibilityIteratorTest, IndexBasedSpellCheckerTest, TestHdfsBackupRestoreCore, TestPullReplicaErrorHandling, SliceStateTest, MetricTriggerIntegrationTest, TestFieldCacheVsDocValues, SolrIndexConfigTest, TestQueryUtils, JWTVerificationkeyResolverTest, DistributedSpellCheckComponentTest, TestRequestStatusCollectionAPI, MissingSegmentRecoveryTest, NodeAddedTriggerTest, TestLuceneIndexBackCompat, TestConfigSetProperties, SolrGraphiteReporterTest, DateFieldTest, RemoteQueryErrorTest, MoveReplicaTest, TestSchemaNameResource, SolrLogAuditLoggerPluginTest, ChaosMonkeySafeLeaderTest, TestMinHashQParser, HealthCheckHandlerTest, TestConfigSetsAPI, EchoParamsTest, TestLeaderElectionZkExpiry, TestSolrJacksonAnnotation, TestSubQueryTransformerDistrib, TestDynamicFieldCollectionResource, TestImpersonationWithHadoopAuth, TestExportWriter, StatelessScriptUpdateProcessorFactoryTest, CdcrOpsAndBoundariesTest, HdfsWriteToMultipleCollectionsTest, TestRTimerTree, SolrSlf4jReporterTest, TestNoOpRegenerator, TestLRUCache, TestFieldResource, CoreAdminRequestStatusTest, DistribCursorPagingTest, SplitByPrefixTest, CursorPagingTest, TestStressRecovery, TestStressInPlaceUpdates, DistributedFacetPivotSmallTest, CollectionReloadTest, DateMathParserTest, TestCloudJSONFacetJoinDomain, PhrasesIdentificationComponentTest, TestDFISimilarityFactory, NodeMarkersRegistrationTest, TestAnalyzedSuggestions, TestReversedWildcardFilterFactory, LukeRequestHandlerTest, MetricsHistoryHandlerTest, ActionThrottleTest, CleanupOldIndexTest, TestPhraseSuggestions, ClusterStateUpdateTest, CoreAdminCreateDiscoverTest, TestSimDistribStateManager, TestGroupingSearch, BJQParserTest, TestDistributedMissingSort, SpellCheckComponentTest, TermsComponentTest, HdfsCollectionsAPIDistributedZkTest, DistributedDebugComponentTest, SolrTestCaseJ4Test, TestCrossCoreJoin, TestCursorMarkWithoutUniqueKey, TestDistributedGrouping, TestDocumentBuilder, TestJoin, TestRandomDVFaceting, TestRandomFaceting, TestSolrCoreProperties, TestSolrTestCaseJ4, TestTolerantSearch, ProtectedTermFilterFactoryTest, TestLuceneMatchVersion, TokenizerChainTest, TestEmbeddedSolrServerConstructors, TestEmbeddedSolrServerSchemaAPI, TestJettySolrRunner, BasicZkTest, ChaosMonkeyNothingIsSafeTest, ClusterStateTest, CollectionStateFormat2Test, DeleteNodeTest, DeleteStatusTest, ForceLeaderTest, ForceLeaderWithTlogReplicasTest, LeaderElectionTest, LeaderFailoverAfterPartitionTest, LeaderTragicEventTest, LeaderVoteWaitTimeoutTest, LegacyCloudClusterPropTest, MetricsHistoryIntegrationTest, MetricsHistoryWithAuthIntegrationTest, MultiThreadedOCPTest, NestedShardedAtomicUpdateTest, NodeMutatorTest, OverriddenZkACLAndCredentialsProvidersTest, OverseerCollectionConfigSetProcessorTest, OverseerRolesTest, OverseerStatusTest, OverseerTaskQueueTest, OverseerTest, PackageManagerCLITest, RecoveryAfterSoftCommitTest, RecoveryZkTest, ReplicationFactorTest, RestartWhileUpdatingTest, SSLMigrationTest, ShardRoutingCustomTest, SolrXmlInZkTest, SplitShardTest, TestCloudInspectUtil, TestCloudPivotFacet, TestCloudRecovery2, TestCryptoKeys, TestDeleteCollectionOnDownNodes, TestDistribDocBasedVersion, TestDownShardTolerantSearch, TestRequestForwarding, TestSSLRandomization, TestSegmentSorting, TestShortCircuitedRequests, TestSizeLimitedDistributedMap, TestSkipOverseerOperations, TestSolrCloudWithKerberosAlt, TestUtilizeNode, UnloadDistributedZkTest, ZkControllerTest, ZkShardTermsTest, CollectionsAPIDistributedZkTest, TestCollectionsAPIViaSolrCloudCluster, NodeAddedTriggerIntegrationTest, NodeLostTriggerTest, ScheduledTriggerIntegrationTest, ScheduledTriggerTest, SearchRateTriggerIntegrationTest]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=SearchRateTriggerIntegrationTest -Dtests.seed=9BD8837E45B27770 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/test-data/enwiki.random.lines.txt -Dtests.locale=ms-MY -Dtests.timezone=Israel -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J0 | SearchRateTriggerIntegrationTest (suite) <<<
   [junit4]    > Throwable #1: junit.framework.AssertionFailedError: Clean up static fields (in @AfterClass?) and null them, your test still has references to classes of which the sizes cannot be measured due to security restrictions or Java 9 module encapsulation:
   [junit4]    >   - private static org.apache.solr.client.solrj.cloud.SolrCloudManager org.apache.solr.cloud.autoscaling.SearchRateTriggerIntegrationTest.cloudManager
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([9BD8837E45B27770]:0)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > Caused by: java.security.AccessControlException: access denied ("java.lang.RuntimePermission" "accessClassInPackage.sun.nio.fs")
   [junit4]    > 	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
   [junit4]    > 	at java.security.AccessController.checkPermission(AccessController.java:884)
   [junit4]    > 	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
   [junit4]    > 	at java.lang.SecurityManager.checkPackageAccess(SecurityManager.java:1564)
   [junit4]    > 	at java.lang.Class.checkPackageAccess(Class.java:2372)
   [junit4]    > 	at java.lang.Class.checkMemberAccess(Class.java:2351)
   [junit4]    > 	at java.lang.Class.getDeclaredFields(Class.java:1915)
   [junit4]    > 	at java.security.AccessController.doPrivileged(Native Method)
   [junit4]    > 	... 10 more
   [junit4] Completed [468/887 (1!)] on J0 in 70.79s, 3 tests, 1 failure, 1 skipped <<< FAILURES!

[...truncated 54432 lines...]

[JENKINS] Lucene-Solr-NightlyTests-8.x - Build # 293 - Still unstable

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-8.x/293/

10 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsWriteToMultipleCollectionsTest

Error Message:
access denied ("java.io.FilePermission" "<<ALL FILES>>" "execute")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "<<ALL FILES>>" "execute")
	at __randomizedtesting.SeedInfo.seed([DCFC7BCF12623A8C]:0)
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.security.AccessController.checkPermission(AccessController.java:884)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkExec(SecurityManager.java:799)
	at java.lang.ProcessBuilder.start(ProcessBuilder.java:1018)
	at org.apache.hadoop.util.Shell.runCommand(Shell.java:938)
	at org.apache.hadoop.util.Shell.run(Shell.java:901)
	at org.apache.hadoop.fs.DF.getFilesystem(DF.java:74)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker$CheckedVolume.<init>(NameNodeResourceChecker.java:70)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker.addDirToCheck(NameNodeResourceChecker.java:166)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker.<init>(NameNodeResourceChecker.java:135)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startCommonServices(FSNamesystem.java:1168)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.startCommonServices(NameNode.java:791)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.initialize(NameNode.java:717)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.<init>(NameNode.java:940)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.<init>(NameNode.java:913)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.createNameNode(NameNode.java:1646)
	at org.apache.hadoop.hdfs.MiniDFSCluster.createNameNode(MiniDFSCluster.java:1314)
	at org.apache.hadoop.hdfs.MiniDFSCluster.configureNameService(MiniDFSCluster.java:1083)
	at org.apache.hadoop.hdfs.MiniDFSCluster.createNameNodesAndSetConf(MiniDFSCluster.java:958)
	at org.apache.hadoop.hdfs.MiniDFSCluster.initMiniDFSCluster(MiniDFSCluster.java:890)
	at org.apache.hadoop.hdfs.MiniDFSCluster.<init>(MiniDFSCluster.java:518)
	at org.apache.hadoop.hdfs.MiniDFSCluster$Builder.build(MiniDFSCluster.java:477)
	at org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:134)
	at org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:66)
	at org.apache.solr.cloud.hdfs.HdfsWriteToMultipleCollectionsTest.setupClass(HdfsWriteToMultipleCollectionsTest.java:67)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:878)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsWriteToMultipleCollectionsTest

Error Message:
15 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsWriteToMultipleCollectionsTest:     1) Thread[id=7499, name=pool-3-thread-1, state=TIMED_WAITING, group=TGRP-HdfsWriteToMultipleCollectionsTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=7507, name=qtp1950460106-7507, state=TIMED_WAITING, group=TGRP-HdfsWriteToMultipleCollectionsTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=7497, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-HdfsWriteToMultipleCollectionsTest]         at java.lang.Object.wait(Native Method)         at java.util.TimerThread.mainLoop(Timer.java:552)         at java.util.TimerThread.run(Timer.java:505)    4) Thread[id=7514, name=FSEditLogAsync, state=WAITING, group=TGRP-HdfsWriteToMultipleCollectionsTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)         at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.dequeueEdit(FSEditLogAsync.java:221)         at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.run(FSEditLogAsync.java:229)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=7504, name=qtp1950460106-7504, state=TIMED_WAITING, group=TGRP-HdfsWriteToMultipleCollectionsTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=7505, name=qtp1950460106-7505, state=TIMED_WAITING, group=TGRP-HdfsWriteToMultipleCollectionsTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    7) Thread[id=7501, name=qtp1950460106-7501, state=RUNNABLE, group=TGRP-HdfsWriteToMultipleCollectionsTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$94/697853099.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)         at java.lang.Thread.run(Thread.java:748)    8) Thread[id=7506, name=qtp1950460106-7506, state=TIMED_WAITING, group=TGRP-HdfsWriteToMultipleCollectionsTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    9) Thread[id=7508, name=Session-HouseKeeper-12384d45, state=TIMED_WAITING, group=TGRP-HdfsWriteToMultipleCollectionsTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   10) Thread[id=7498, name=org.apache.hadoop.util.JvmPauseMonitor$Monitor@408e55d2, state=TIMED_WAITING, group=TGRP-HdfsWriteToMultipleCollectionsTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192)         at java.lang.Thread.run(Thread.java:748)   11) Thread[id=7517, name=IPC Server idle connection scanner for port 45811, state=WAITING, group=TGRP-HdfsWriteToMultipleCollectionsTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at java.util.TimerThread.mainLoop(Timer.java:526)         at java.util.TimerThread.run(Timer.java:505)   12) Thread[id=7502, name=qtp1950460106-7502-acceptor-0@57f130a3-ServerConnector@10c3ba90{HTTP/1.1,[http/1.1]}{localhost.localdomain:40439}, state=RUNNABLE, group=TGRP-HdfsWriteToMultipleCollectionsTest]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)         at java.lang.Thread.run(Thread.java:748)   13) Thread[id=7500, name=qtp1950460106-7500, state=RUNNABLE, group=TGRP-HdfsWriteToMultipleCollectionsTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$94/697853099.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)         at java.lang.Thread.run(Thread.java:748)   14) Thread[id=7503, name=qtp1950460106-7503, state=TIMED_WAITING, group=TGRP-HdfsWriteToMultipleCollectionsTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)   15) Thread[id=7516, name=Socket Reader #1 for port 45811, state=RUNNABLE, group=TGRP-HdfsWriteToMultipleCollectionsTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097)         at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 15 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsWriteToMultipleCollectionsTest: 
   1) Thread[id=7499, name=pool-3-thread-1, state=TIMED_WAITING, group=TGRP-HdfsWriteToMultipleCollectionsTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=7507, name=qtp1950460106-7507, state=TIMED_WAITING, group=TGRP-HdfsWriteToMultipleCollectionsTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=7497, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-HdfsWriteToMultipleCollectionsTest]
        at java.lang.Object.wait(Native Method)
        at java.util.TimerThread.mainLoop(Timer.java:552)
        at java.util.TimerThread.run(Timer.java:505)
   4) Thread[id=7514, name=FSEditLogAsync, state=WAITING, group=TGRP-HdfsWriteToMultipleCollectionsTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)
        at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.dequeueEdit(FSEditLogAsync.java:221)
        at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.run(FSEditLogAsync.java:229)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=7504, name=qtp1950460106-7504, state=TIMED_WAITING, group=TGRP-HdfsWriteToMultipleCollectionsTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=7505, name=qtp1950460106-7505, state=TIMED_WAITING, group=TGRP-HdfsWriteToMultipleCollectionsTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   7) Thread[id=7501, name=qtp1950460106-7501, state=RUNNABLE, group=TGRP-HdfsWriteToMultipleCollectionsTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$94/697853099.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
        at java.lang.Thread.run(Thread.java:748)
   8) Thread[id=7506, name=qtp1950460106-7506, state=TIMED_WAITING, group=TGRP-HdfsWriteToMultipleCollectionsTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   9) Thread[id=7508, name=Session-HouseKeeper-12384d45, state=TIMED_WAITING, group=TGRP-HdfsWriteToMultipleCollectionsTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  10) Thread[id=7498, name=org.apache.hadoop.util.JvmPauseMonitor$Monitor@408e55d2, state=TIMED_WAITING, group=TGRP-HdfsWriteToMultipleCollectionsTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192)
        at java.lang.Thread.run(Thread.java:748)
  11) Thread[id=7517, name=IPC Server idle connection scanner for port 45811, state=WAITING, group=TGRP-HdfsWriteToMultipleCollectionsTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at java.util.TimerThread.mainLoop(Timer.java:526)
        at java.util.TimerThread.run(Timer.java:505)
  12) Thread[id=7502, name=qtp1950460106-7502-acceptor-0@57f130a3-ServerConnector@10c3ba90{HTTP/1.1,[http/1.1]}{localhost.localdomain:40439}, state=RUNNABLE, group=TGRP-HdfsWriteToMultipleCollectionsTest]
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
        at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385)
        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
        at java.lang.Thread.run(Thread.java:748)
  13) Thread[id=7500, name=qtp1950460106-7500, state=RUNNABLE, group=TGRP-HdfsWriteToMultipleCollectionsTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$94/697853099.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
        at java.lang.Thread.run(Thread.java:748)
  14) Thread[id=7503, name=qtp1950460106-7503, state=TIMED_WAITING, group=TGRP-HdfsWriteToMultipleCollectionsTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
  15) Thread[id=7516, name=Socket Reader #1 for port 45811, state=RUNNABLE, group=TGRP-HdfsWriteToMultipleCollectionsTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097)
        at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076)
	at __randomizedtesting.SeedInfo.seed([DCFC7BCF12623A8C]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsWriteToMultipleCollectionsTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=7499, name=pool-3-thread-1, state=TIMED_WAITING, group=TGRP-HdfsWriteToMultipleCollectionsTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=7507, name=qtp1950460106-7507, state=TIMED_WAITING, group=TGRP-HdfsWriteToMultipleCollectionsTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=7497, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-HdfsWriteToMultipleCollectionsTest]         at java.lang.Object.wait(Native Method)         at java.util.TimerThread.mainLoop(Timer.java:552)         at java.util.TimerThread.run(Timer.java:505)    4) Thread[id=7504, name=qtp1950460106-7504, state=TIMED_WAITING, group=TGRP-HdfsWriteToMultipleCollectionsTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=7505, name=qtp1950460106-7505, state=TIMED_WAITING, group=TGRP-HdfsWriteToMultipleCollectionsTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=7501, name=qtp1950460106-7501, state=RUNNABLE, group=TGRP-HdfsWriteToMultipleCollectionsTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$94/697853099.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)         at java.lang.Thread.run(Thread.java:748)    7) Thread[id=7506, name=qtp1950460106-7506, state=TIMED_WAITING, group=TGRP-HdfsWriteToMultipleCollectionsTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    8) Thread[id=7508, name=Session-HouseKeeper-12384d45, state=TIMED_WAITING, group=TGRP-HdfsWriteToMultipleCollectionsTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    9) Thread[id=7517, name=IPC Server idle connection scanner for port 45811, state=WAITING, group=TGRP-HdfsWriteToMultipleCollectionsTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at java.util.TimerThread.mainLoop(Timer.java:526)         at java.util.TimerThread.run(Timer.java:505)   10) Thread[id=7502, name=qtp1950460106-7502, state=TIMED_WAITING, group=TGRP-HdfsWriteToMultipleCollectionsTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)   11) Thread[id=7500, name=qtp1950460106-7500, state=RUNNABLE, group=TGRP-HdfsWriteToMultipleCollectionsTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$94/697853099.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)         at java.lang.Thread.run(Thread.java:748)   12) Thread[id=7503, name=qtp1950460106-7503, state=TIMED_WAITING, group=TGRP-HdfsWriteToMultipleCollectionsTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)   13) Thread[id=7516, name=Socket Reader #1 for port 45811, state=RUNNABLE, group=TGRP-HdfsWriteToMultipleCollectionsTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097)         at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=7499, name=pool-3-thread-1, state=TIMED_WAITING, group=TGRP-HdfsWriteToMultipleCollectionsTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=7507, name=qtp1950460106-7507, state=TIMED_WAITING, group=TGRP-HdfsWriteToMultipleCollectionsTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=7497, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-HdfsWriteToMultipleCollectionsTest]
        at java.lang.Object.wait(Native Method)
        at java.util.TimerThread.mainLoop(Timer.java:552)
        at java.util.TimerThread.run(Timer.java:505)
   4) Thread[id=7504, name=qtp1950460106-7504, state=TIMED_WAITING, group=TGRP-HdfsWriteToMultipleCollectionsTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=7505, name=qtp1950460106-7505, state=TIMED_WAITING, group=TGRP-HdfsWriteToMultipleCollectionsTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=7501, name=qtp1950460106-7501, state=RUNNABLE, group=TGRP-HdfsWriteToMultipleCollectionsTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$94/697853099.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
        at java.lang.Thread.run(Thread.java:748)
   7) Thread[id=7506, name=qtp1950460106-7506, state=TIMED_WAITING, group=TGRP-HdfsWriteToMultipleCollectionsTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   8) Thread[id=7508, name=Session-HouseKeeper-12384d45, state=TIMED_WAITING, group=TGRP-HdfsWriteToMultipleCollectionsTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   9) Thread[id=7517, name=IPC Server idle connection scanner for port 45811, state=WAITING, group=TGRP-HdfsWriteToMultipleCollectionsTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at java.util.TimerThread.mainLoop(Timer.java:526)
        at java.util.TimerThread.run(Timer.java:505)
  10) Thread[id=7502, name=qtp1950460106-7502, state=TIMED_WAITING, group=TGRP-HdfsWriteToMultipleCollectionsTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
  11) Thread[id=7500, name=qtp1950460106-7500, state=RUNNABLE, group=TGRP-HdfsWriteToMultipleCollectionsTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$94/697853099.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
        at java.lang.Thread.run(Thread.java:748)
  12) Thread[id=7503, name=qtp1950460106-7503, state=TIMED_WAITING, group=TGRP-HdfsWriteToMultipleCollectionsTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
  13) Thread[id=7516, name=Socket Reader #1 for port 45811, state=RUNNABLE, group=TGRP-HdfsWriteToMultipleCollectionsTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097)
        at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076)
	at __randomizedtesting.SeedInfo.seed([DCFC7BCF12623A8C]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.security.hadoop.TestImpersonationWithHadoopAuth

Error Message:
java.security.AccessControlException: access denied ("java.io.FilePermission" "<<ALL FILES>>" "execute")

Stack Trace:
com.google.common.util.concurrent.UncheckedExecutionException: java.security.AccessControlException: access denied ("java.io.FilePermission" "<<ALL FILES>>" "execute")
	at __randomizedtesting.SeedInfo.seed([DCFC7BCF12623A8C]:0)
	at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2050)
	at com.google.common.cache.LocalCache.get(LocalCache.java:3951)
	at com.google.common.cache.LocalCache.getOrLoad(LocalCache.java:3973)
	at com.google.common.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4957)
	at org.apache.hadoop.security.Groups.getGroups(Groups.java:228)
	at org.apache.solr.security.hadoop.ImpersonationUtil.getUsersFirstGroup(ImpersonationUtil.java:42)
	at org.apache.solr.security.hadoop.TestImpersonationWithHadoopAuth.setupClass(TestImpersonationWithHadoopAuth.java:74)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:878)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.security.AccessControlException: access denied ("java.io.FilePermission" "<<ALL FILES>>" "execute")
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.security.AccessController.checkPermission(AccessController.java:884)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkExec(SecurityManager.java:799)
	at java.lang.ProcessBuilder.start(ProcessBuilder.java:1018)
	at org.apache.hadoop.util.Shell.runCommand(Shell.java:938)
	at org.apache.hadoop.util.Shell.run(Shell.java:901)
	at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1213)
	at org.apache.hadoop.security.ShellBasedUnixGroupsMapping.getUnixGroups(ShellBasedUnixGroupsMapping.java:200)
	at org.apache.hadoop.security.ShellBasedUnixGroupsMapping.getGroups(ShellBasedUnixGroupsMapping.java:97)
	at org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback.getGroups(JniBasedUnixGroupsMappingWithFallback.java:51)
	at org.apache.hadoop.security.Groups$GroupCacheLoader.fetchGroupList(Groups.java:387)
	at org.apache.hadoop.security.Groups$GroupCacheLoader.load(Groups.java:321)
	at org.apache.hadoop.security.Groups$GroupCacheLoader.load(Groups.java:270)
	at com.google.common.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3527)
	at com.google.common.cache.LocalCache$Segment.loadSync(LocalCache.java:2276)
	at com.google.common.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2154)
	at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2044)
	... 30 more


FAILED:  junit.framework.TestSuite.org.apache.solr.store.hdfs.HdfsDirectoryTest

Error Message:
access denied ("java.io.FilePermission" "<<ALL FILES>>" "execute")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "<<ALL FILES>>" "execute")
	at __randomizedtesting.SeedInfo.seed([DCFC7BCF12623A8C]:0)
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.security.AccessController.checkPermission(AccessController.java:884)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkExec(SecurityManager.java:799)
	at java.lang.ProcessBuilder.start(ProcessBuilder.java:1018)
	at org.apache.hadoop.util.Shell.runCommand(Shell.java:938)
	at org.apache.hadoop.util.Shell.run(Shell.java:901)
	at org.apache.hadoop.fs.DF.getFilesystem(DF.java:74)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker$CheckedVolume.<init>(NameNodeResourceChecker.java:70)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker.addDirToCheck(NameNodeResourceChecker.java:166)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker.<init>(NameNodeResourceChecker.java:135)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startCommonServices(FSNamesystem.java:1168)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.startCommonServices(NameNode.java:791)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.initialize(NameNode.java:717)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.<init>(NameNode.java:940)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.<init>(NameNode.java:913)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.createNameNode(NameNode.java:1646)
	at org.apache.hadoop.hdfs.MiniDFSCluster.createNameNode(MiniDFSCluster.java:1314)
	at org.apache.hadoop.hdfs.MiniDFSCluster.configureNameService(MiniDFSCluster.java:1083)
	at org.apache.hadoop.hdfs.MiniDFSCluster.createNameNodesAndSetConf(MiniDFSCluster.java:958)
	at org.apache.hadoop.hdfs.MiniDFSCluster.initMiniDFSCluster(MiniDFSCluster.java:890)
	at org.apache.hadoop.hdfs.MiniDFSCluster.<init>(MiniDFSCluster.java:518)
	at org.apache.hadoop.hdfs.MiniDFSCluster$Builder.build(MiniDFSCluster.java:477)
	at org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:134)
	at org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:66)
	at org.apache.solr.store.hdfs.HdfsDirectoryTest.beforeClass(HdfsDirectoryTest.java:60)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:878)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.store.hdfs.HdfsDirectoryTest

Error Message:
15 threads leaked from SUITE scope at org.apache.solr.store.hdfs.HdfsDirectoryTest:     1) Thread[id=13952, name=IPC Server idle connection scanner for port 42279, state=WAITING, group=TGRP-HdfsDirectoryTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at java.util.TimerThread.mainLoop(Timer.java:526)         at java.util.TimerThread.run(Timer.java:505)    2) Thread[id=13937, name=qtp649617958-13937-acceptor-0@41978334-ServerConnector@4ee68d4e{HTTP/1.1,[http/1.1]}{localhost.localdomain:38339}, state=RUNNABLE, group=TGRP-HdfsDirectoryTest]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=13938, name=qtp649617958-13938, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=13943, name=Session-HouseKeeper-e88710a, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=13942, name=qtp649617958-13942, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=13932, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest]         at java.lang.Object.wait(Native Method)         at java.util.TimerThread.mainLoop(Timer.java:552)         at java.util.TimerThread.run(Timer.java:505)    7) Thread[id=13949, name=FSEditLogAsync, state=WAITING, group=TGRP-HdfsDirectoryTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)         at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.dequeueEdit(FSEditLogAsync.java:221)         at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.run(FSEditLogAsync.java:229)         at java.lang.Thread.run(Thread.java:748)    8) Thread[id=13933, name=org.apache.hadoop.util.JvmPauseMonitor$Monitor@649e787f, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192)         at java.lang.Thread.run(Thread.java:748)    9) Thread[id=13951, name=Socket Reader #1 for port 42279, state=RUNNABLE, group=TGRP-HdfsDirectoryTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097)         at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076)   10) Thread[id=13940, name=qtp649617958-13940, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)   11) Thread[id=13935, name=qtp649617958-13935, state=RUNNABLE, group=TGRP-HdfsDirectoryTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$46/509488391.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)         at java.lang.Thread.run(Thread.java:748)   12) Thread[id=13936, name=qtp649617958-13936, state=RUNNABLE, group=TGRP-HdfsDirectoryTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$46/509488391.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)         at java.lang.Thread.run(Thread.java:748)   13) Thread[id=13939, name=qtp649617958-13939, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)   14) Thread[id=13934, name=pool-5-thread-1, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   15) Thread[id=13941, name=qtp649617958-13941, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 15 threads leaked from SUITE scope at org.apache.solr.store.hdfs.HdfsDirectoryTest: 
   1) Thread[id=13952, name=IPC Server idle connection scanner for port 42279, state=WAITING, group=TGRP-HdfsDirectoryTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at java.util.TimerThread.mainLoop(Timer.java:526)
        at java.util.TimerThread.run(Timer.java:505)
   2) Thread[id=13937, name=qtp649617958-13937-acceptor-0@41978334-ServerConnector@4ee68d4e{HTTP/1.1,[http/1.1]}{localhost.localdomain:38339}, state=RUNNABLE, group=TGRP-HdfsDirectoryTest]
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
        at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385)
        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=13938, name=qtp649617958-13938, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=13943, name=Session-HouseKeeper-e88710a, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=13942, name=qtp649617958-13942, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=13932, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest]
        at java.lang.Object.wait(Native Method)
        at java.util.TimerThread.mainLoop(Timer.java:552)
        at java.util.TimerThread.run(Timer.java:505)
   7) Thread[id=13949, name=FSEditLogAsync, state=WAITING, group=TGRP-HdfsDirectoryTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)
        at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.dequeueEdit(FSEditLogAsync.java:221)
        at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.run(FSEditLogAsync.java:229)
        at java.lang.Thread.run(Thread.java:748)
   8) Thread[id=13933, name=org.apache.hadoop.util.JvmPauseMonitor$Monitor@649e787f, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192)
        at java.lang.Thread.run(Thread.java:748)
   9) Thread[id=13951, name=Socket Reader #1 for port 42279, state=RUNNABLE, group=TGRP-HdfsDirectoryTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097)
        at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076)
  10) Thread[id=13940, name=qtp649617958-13940, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
  11) Thread[id=13935, name=qtp649617958-13935, state=RUNNABLE, group=TGRP-HdfsDirectoryTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$46/509488391.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
        at java.lang.Thread.run(Thread.java:748)
  12) Thread[id=13936, name=qtp649617958-13936, state=RUNNABLE, group=TGRP-HdfsDirectoryTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$46/509488391.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
        at java.lang.Thread.run(Thread.java:748)
  13) Thread[id=13939, name=qtp649617958-13939, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
  14) Thread[id=13934, name=pool-5-thread-1, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  15) Thread[id=13941, name=qtp649617958-13941, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([DCFC7BCF12623A8C]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.store.hdfs.HdfsDirectoryTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=13952, name=IPC Server idle connection scanner for port 42279, state=WAITING, group=TGRP-HdfsDirectoryTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at java.util.TimerThread.mainLoop(Timer.java:526)         at java.util.TimerThread.run(Timer.java:505)    2) Thread[id=13937, name=qtp649617958-13937, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=13938, name=qtp649617958-13938, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=13943, name=Session-HouseKeeper-e88710a, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=13942, name=qtp649617958-13942, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=13932, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest]         at java.lang.Object.wait(Native Method)         at java.util.TimerThread.mainLoop(Timer.java:552)         at java.util.TimerThread.run(Timer.java:505)    7) Thread[id=13951, name=Socket Reader #1 for port 42279, state=RUNNABLE, group=TGRP-HdfsDirectoryTest]         at sun.nio.ch.EPollArrayWrapper.interrupt(Native Method)         at sun.nio.ch.EPollArrayWrapper.interrupt(EPollArrayWrapper.java:317)         at sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:207)         at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)         at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097)         at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076)    8) Thread[id=13940, name=qtp649617958-13940, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    9) Thread[id=13935, name=qtp649617958-13935, state=RUNNABLE, group=TGRP-HdfsDirectoryTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$46/509488391.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)         at java.lang.Thread.run(Thread.java:748)   10) Thread[id=13936, name=qtp649617958-13936, state=RUNNABLE, group=TGRP-HdfsDirectoryTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.ja

[...truncated too long message...]

uledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    5) Thread[id=13942, name=qtp649617958-13942, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    6) Thread[id=13932, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest]
   [junit4]    >         at java.lang.Object.wait(Native Method)
   [junit4]    >         at java.util.TimerThread.mainLoop(Timer.java:552)
   [junit4]    >         at java.util.TimerThread.run(Timer.java:505)
   [junit4]    >    7) Thread[id=13951, name=Socket Reader #1 for port 42279, state=RUNNABLE, group=TGRP-HdfsDirectoryTest]
   [junit4]    >         at sun.nio.ch.EPollArrayWrapper.interrupt(Native Method)
   [junit4]    >         at sun.nio.ch.EPollArrayWrapper.interrupt(EPollArrayWrapper.java:317)
   [junit4]    >         at sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:207)
   [junit4]    >         at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
   [junit4]    >         at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
   [junit4]    >         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92)
   [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
   [junit4]    >         at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097)
   [junit4]    >         at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076)
   [junit4]    >    8) Thread[id=13940, name=qtp649617958-13940, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    9) Thread[id=13935, name=qtp649617958-13935, state=RUNNABLE, group=TGRP-HdfsDirectoryTest]
   [junit4]    >         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
   [junit4]    >         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
   [junit4]    >         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
   [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$$Lambda$46/509488391.run(Unknown Source)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   10) Thread[id=13936, name=qtp649617958-13936, state=RUNNABLE, group=TGRP-HdfsDirectoryTest]
   [junit4]    >         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
   [junit4]    >         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
   [junit4]    >         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
   [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$$Lambda$46/509488391.run(Unknown Source)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   11) Thread[id=13939, name=qtp649617958-13939, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   12) Thread[id=13934, name=pool-5-thread-1, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   13) Thread[id=13941, name=qtp649617958-13941, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([DCFC7BCF12623A8C]:0)
   [junit4] Completed [770/887 (4!)] on J1 in 15.22s, 0 tests, 3 errors <<< FAILURES!

[...truncated 53279 lines...]

[JENKINS] Lucene-Solr-NightlyTests-8.x - Build # 292 - Failure

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-8.x/292/

10 tests failed.
FAILED:  org.apache.lucene.document.TestXYMultiPolygonShapeQueries.testRandomBig

Error Message:
GC overhead limit exceeded

Stack Trace:
java.lang.OutOfMemoryError: GC overhead limit exceeded
	at __randomizedtesting.SeedInfo.seed([308E6872818E62A9:B7D915FD10D71E29]:0)
	at org.apache.lucene.store.RAMFile.newBuffer(RAMFile.java:84)
	at org.apache.lucene.store.RAMFile.addBuffer(RAMFile.java:57)
	at org.apache.lucene.store.RAMOutputStream.switchCurrentBuffer(RAMOutputStream.java:168)
	at org.apache.lucene.store.RAMOutputStream.writeBytes(RAMOutputStream.java:154)
	at org.apache.lucene.store.MockIndexOutputWrapper.writeBytes(MockIndexOutputWrapper.java:141)
	at org.apache.lucene.util.bkd.OfflinePointWriter.append(OfflinePointWriter.java:67)
	at org.apache.lucene.util.bkd.BKDRadixSelector.offlinePartition(BKDRadixSelector.java:285)
	at org.apache.lucene.util.bkd.BKDRadixSelector.buildHistogramAndPartition(BKDRadixSelector.java:258)
	at org.apache.lucene.util.bkd.BKDRadixSelector.select(BKDRadixSelector.java:126)
	at org.apache.lucene.util.bkd.BKDWriter.build(BKDWriter.java:1647)
	at org.apache.lucene.util.bkd.BKDWriter.build(BKDWriter.java:1664)
	at org.apache.lucene.util.bkd.BKDWriter.build(BKDWriter.java:1664)
	at org.apache.lucene.util.bkd.BKDWriter.finish(BKDWriter.java:797)
	at org.apache.lucene.index.RandomCodec$1$1.writeField(RandomCodec.java:140)
	at org.apache.lucene.codecs.PointsWriter.mergeOneField(PointsWriter.java:62)
	at org.apache.lucene.codecs.PointsWriter.merge(PointsWriter.java:191)
	at org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.merge(Lucene60PointsWriter.java:143)
	at org.apache.lucene.codecs.asserting.AssertingPointsFormat$AssertingPointsWriter.merge(AssertingPointsFormat.java:149)
	at org.apache.lucene.index.SegmentMerger.mergePoints(SegmentMerger.java:202)
	at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:162)
	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
	at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
	at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2158)
	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1991)
	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1940)
	at org.apache.lucene.document.BaseShapeTestCase.indexRandomShapes(BaseShapeTestCase.java:242)
	at org.apache.lucene.document.BaseShapeTestCase.verify(BaseShapeTestCase.java:211)
	at org.apache.lucene.document.BaseShapeTestCase.doTestRandom(BaseShapeTestCase.java:137)
	at org.apache.lucene.document.TestXYMultiPolygonShapeQueries.testRandomBig(TestXYMultiPolygonShapeQueries.java:124)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.MoveReplicaHDFSFailoverTest

Error Message:
access denied ("java.io.FilePermission" "<<ALL FILES>>" "execute")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "<<ALL FILES>>" "execute")
	at __randomizedtesting.SeedInfo.seed([AA80BF4650F470FD]:0)
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.security.AccessController.checkPermission(AccessController.java:884)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkExec(SecurityManager.java:799)
	at java.lang.ProcessBuilder.start(ProcessBuilder.java:1018)
	at org.apache.hadoop.util.Shell.runCommand(Shell.java:938)
	at org.apache.hadoop.util.Shell.run(Shell.java:901)
	at org.apache.hadoop.fs.DF.getFilesystem(DF.java:74)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker$CheckedVolume.<init>(NameNodeResourceChecker.java:70)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker.addDirToCheck(NameNodeResourceChecker.java:166)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker.<init>(NameNodeResourceChecker.java:135)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startCommonServices(FSNamesystem.java:1168)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.startCommonServices(NameNode.java:791)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.initialize(NameNode.java:717)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.<init>(NameNode.java:940)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.<init>(NameNode.java:913)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.createNameNode(NameNode.java:1646)
	at org.apache.hadoop.hdfs.MiniDFSCluster.createNameNode(MiniDFSCluster.java:1314)
	at org.apache.hadoop.hdfs.MiniDFSCluster.configureNameService(MiniDFSCluster.java:1083)
	at org.apache.hadoop.hdfs.MiniDFSCluster.createNameNodesAndSetConf(MiniDFSCluster.java:958)
	at org.apache.hadoop.hdfs.MiniDFSCluster.initMiniDFSCluster(MiniDFSCluster.java:890)
	at org.apache.hadoop.hdfs.MiniDFSCluster.<init>(MiniDFSCluster.java:518)
	at org.apache.hadoop.hdfs.MiniDFSCluster$Builder.build(MiniDFSCluster.java:477)
	at org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:134)
	at org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:66)
	at org.apache.solr.cloud.MoveReplicaHDFSFailoverTest.setupClass(MoveReplicaHDFSFailoverTest.java:55)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:878)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.MoveReplicaHDFSFailoverTest

Error Message:
15 threads leaked from SUITE scope at org.apache.solr.cloud.MoveReplicaHDFSFailoverTest:     1) Thread[id=2410, name=qtp1071598924-2410, state=RUNNABLE, group=TGRP-MoveReplicaHDFSFailoverTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$59/1585840971.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=2414, name=qtp1071598924-2414, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=2408, name=pool-3-thread-1, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=2407, name=org.apache.hadoop.util.JvmPauseMonitor$Monitor@194f0213, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=2412, name=qtp1071598924-2412, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=2416, name=qtp1071598924-2416, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    7) Thread[id=2413, name=qtp1071598924-2413, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    8) Thread[id=2411, name=qtp1071598924-2411-acceptor-0@2fb73beb-ServerConnector@4bb4d0d2{HTTP/1.1,[http/1.1]}{localhost.localdomain:40137}, state=RUNNABLE, group=TGRP-MoveReplicaHDFSFailoverTest]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)         at java.lang.Thread.run(Thread.java:748)    9) Thread[id=2425, name=Socket Reader #1 for port 46071, state=RUNNABLE, group=TGRP-MoveReplicaHDFSFailoverTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097)         at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076)   10) Thread[id=2415, name=qtp1071598924-2415, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)   11) Thread[id=2409, name=qtp1071598924-2409, state=RUNNABLE, group=TGRP-MoveReplicaHDFSFailoverTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$59/1585840971.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)         at java.lang.Thread.run(Thread.java:748)   12) Thread[id=2426, name=IPC Server idle connection scanner for port 46071, state=WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at java.util.TimerThread.mainLoop(Timer.java:526)         at java.util.TimerThread.run(Timer.java:505)   13) Thread[id=2406, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]         at java.lang.Object.wait(Native Method)         at java.util.TimerThread.mainLoop(Timer.java:552)         at java.util.TimerThread.run(Timer.java:505)   14) Thread[id=2417, name=Session-HouseKeeper-3e3d8832, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   15) Thread[id=2423, name=FSEditLogAsync, state=WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)         at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.dequeueEdit(FSEditLogAsync.java:221)         at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.run(FSEditLogAsync.java:229)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 15 threads leaked from SUITE scope at org.apache.solr.cloud.MoveReplicaHDFSFailoverTest: 
   1) Thread[id=2410, name=qtp1071598924-2410, state=RUNNABLE, group=TGRP-MoveReplicaHDFSFailoverTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$59/1585840971.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=2414, name=qtp1071598924-2414, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=2408, name=pool-3-thread-1, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=2407, name=org.apache.hadoop.util.JvmPauseMonitor$Monitor@194f0213, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=2412, name=qtp1071598924-2412, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=2416, name=qtp1071598924-2416, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   7) Thread[id=2413, name=qtp1071598924-2413, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   8) Thread[id=2411, name=qtp1071598924-2411-acceptor-0@2fb73beb-ServerConnector@4bb4d0d2{HTTP/1.1,[http/1.1]}{localhost.localdomain:40137}, state=RUNNABLE, group=TGRP-MoveReplicaHDFSFailoverTest]
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
        at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385)
        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
        at java.lang.Thread.run(Thread.java:748)
   9) Thread[id=2425, name=Socket Reader #1 for port 46071, state=RUNNABLE, group=TGRP-MoveReplicaHDFSFailoverTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097)
        at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076)
  10) Thread[id=2415, name=qtp1071598924-2415, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
  11) Thread[id=2409, name=qtp1071598924-2409, state=RUNNABLE, group=TGRP-MoveReplicaHDFSFailoverTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$59/1585840971.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
        at java.lang.Thread.run(Thread.java:748)
  12) Thread[id=2426, name=IPC Server idle connection scanner for port 46071, state=WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at java.util.TimerThread.mainLoop(Timer.java:526)
        at java.util.TimerThread.run(Timer.java:505)
  13) Thread[id=2406, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]
        at java.lang.Object.wait(Native Method)
        at java.util.TimerThread.mainLoop(Timer.java:552)
        at java.util.TimerThread.run(Timer.java:505)
  14) Thread[id=2417, name=Session-HouseKeeper-3e3d8832, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  15) Thread[id=2423, name=FSEditLogAsync, state=WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)
        at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.dequeueEdit(FSEditLogAsync.java:221)
        at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.run(FSEditLogAsync.java:229)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([AA80BF4650F470FD]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.MoveReplicaHDFSFailoverTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=2410, name=qtp1071598924-2410, state=RUNNABLE, group=TGRP-MoveReplicaHDFSFailoverTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$59/1585840971.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=2414, name=qtp1071598924-2414, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=2408, name=pool-3-thread-1, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=2412, name=qtp1071598924-2412, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=2416, name=qtp1071598924-2416, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=2413, name=qtp1071598924-2413, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    7) Thread[id=2411, name=qtp1071598924-2411, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    8) Thread[id=2425, name=Socket Reader #1 for port 46071, state=RUNNABLE, group=TGRP-MoveReplicaHDFSFailoverTest]         at sun.nio.ch.EPollArrayWrapper.interrupt(Native Method)         at sun.nio.ch.EPollArrayWrapper.interrupt(EPollArrayWrapper.java:317)         at sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:207)         at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)         at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097)         at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076)    9) Thread[id=2415, name=qtp1071598924-2415, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)   10) Thread[id=2409, name=qtp1071598924-2409, state=RUNNABLE, group=TGRP-MoveReplicaHDFSFailoverTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$59/1585840971.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)         at java.lang.Thread.run(Thread.java:748)   11) Thread[id=2426, name=IPC Server idle connection scanner for port 46071, state=WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at java.util.TimerThread.mainLoop(Timer.java:526)         at java.util.TimerThread.run(Timer.java:505)   12) Thread[id=2406, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]         at java.lang.Object.wait(Native Method)         at java.util.TimerThread.mainLoop(Timer.java:552)         at java.util.TimerThread.run(Timer.java:505)   13) Thread[id=2417, name=Session-HouseKeeper-3e3d8832, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=2410, name=qtp1071598924-2410, state=RUNNABLE, group=TGRP-MoveReplicaHDFSFailoverTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$59/1585840971.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=2414, name=qtp1071598924-2414, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=2408, name=pool-3-thread-1, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=2412, name=qtp1071598924-2412, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=2416, name=qtp1071598924-2416, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=2413, name=qtp1071598924-2413, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   7) Thread[id=2411, name=qtp1071598924-2411, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   8) Thread[id=2425, name=Socket Reader #1 for port 46071, state=RUNNABLE, group=TGRP-MoveReplicaHDFSFailoverTest]
        at sun.nio.ch.EPollArrayWrapper.interrupt(Native Method)
        at sun.nio.ch.EPollArrayWrapper.interrupt(EPollArrayWrapper.java:317)
        at sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:207)
        at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
        at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097)
        at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076)
   9) Thread[id=2415, name=qtp1071598924-2415, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
  10) Thread[id=2409, name=qtp1071598924-2409, state=RUNNABLE, group=TGRP-MoveReplicaHDFSFailoverTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$59/1585840971.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
        at java.lang.Thread.run(Thread.java:748)
  11) Thread[id=2426, name=IPC Server idle connection scanner for port 46071, state=WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at java.util.TimerThread.mainLoop(Timer.java:526)
        at java.util.TimerThread.run(Timer.java:505)
  12) Thread[id=2406, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]
        at java.lang.Object.wait(Native Method)
        at java.util.TimerThread.mainLoop(Timer.java:552)
        at java.util.TimerThread.run(Timer.java:505)
  13) Thread[id=2417, name=Session-HouseKeeper-3e3d8832, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([AA80BF4650F470FD]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.api.collections.TestHdfsCloudBackupRestore

Error Message:
access denied ("java.io.FilePermission" "<<ALL FILES>>" "execute")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "<<ALL FILES>>" "execute")
	at __randomizedtesting.SeedInfo.seed([AA80BF4650F470FD]:0)
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.security.AccessController.checkPermission(AccessController.java:884)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkExec(SecurityManager.java:799)
	at java.lang.ProcessBuilder.start(ProcessBuilder.java:1018)
	at org.apache.hadoop.util.Shell.runCommand(Shell.java:938)
	at org.apache.hadoop.util.Shell.run(Shell.java:901)
	at org.apache.hadoop.fs.DF.getFilesystem(DF.java:74)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker$CheckedVolume.<init>(NameNodeResourceChecker.java:70)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker.addDirToCheck(NameNodeResourceChecker.java:166)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker.<init>(NameNodeResourceChecker.java:135)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startCommonServices(FSNamesystem.java:1168)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.startCommonServices(NameNode.java:791)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.initialize(NameNode.java:717)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.<init>(NameNode.java:940)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.<init>(NameNode.java:913)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.createNameNode(NameNode.java:1646)
	at org.apache.hadoop.hdfs.MiniDFSCluster.createNameNode(MiniDFSCluster.java:1314)
	at org.apache.hadoop.hdfs.MiniDFSCluster.configureNameService(MiniDFSCluster.java:1083)
	at org.apache.hadoop.hdfs.MiniDFSCluster.createNameNodesAndSetConf(MiniDFSCluster.java:958)
	at org.apache.hadoop.hdfs.MiniDFSCluster.initMiniDFSCluster(MiniDFSCluster.java:890)
	at org.apache.hadoop.hdfs.MiniDFSCluster.<init>(MiniDFSCluster.java:518)
	at org.apache.hadoop.hdfs.MiniDFSCluster$Builder.build(MiniDFSCluster.java:477)
	at org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:134)
	at org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:66)
	at org.apache.solr.cloud.api.collections.TestHdfsCloudBackupRestore.setupClass(TestHdfsCloudBackupRestore.java:110)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:878)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.api.collections.TestHdfsCloudBackupRestore

Error Message:
15 threads leaked from SUITE scope at org.apache.solr.cloud.api.collections.TestHdfsCloudBackupRestore:     1) Thread[id=5871, name=Session-HouseKeeper-2eb24c26, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=5867, name=qtp1033731312-5867, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=5860, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]         at java.lang.Object.wait(Native Method)         at java.util.TimerThread.mainLoop(Timer.java:552)         at java.util.TimerThread.run(Timer.java:505)    4) Thread[id=5877, name=FSEditLogAsync, state=WAITING, group=TGRP-TestHdfsCloudBackupRestore]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)         at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.dequeueEdit(FSEditLogAsync.java:221)         at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.run(FSEditLogAsync.java:229)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=5866, name=qtp1033731312-5866, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=5869, name=qtp1033731312-5869, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    7) Thread[id=5870, name=qtp1033731312-5870, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    8) Thread[id=5868, name=qtp1033731312-5868, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    9) Thread[id=5879, name=Socket Reader #1 for port 45315, state=RUNNABLE, group=TGRP-TestHdfsCloudBackupRestore]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097)         at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076)   10) Thread[id=5861, name=org.apache.hadoop.util.JvmPauseMonitor$Monitor@1dcd311d, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]         at java.lang.Thread.sleep(Native Method)         at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192)         at java.lang.Thread.run(Thread.java:748)   11) Thread[id=5863, name=qtp1033731312-5863, state=RUNNABLE, group=TGRP-TestHdfsCloudBackupRestore]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$58/1990976794.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)         at java.lang.Thread.run(Thread.java:748)   12) Thread[id=5865, name=qtp1033731312-5865-acceptor-0@57366549-ServerConnector@7e301298{HTTP/1.1,[http/1.1]}{localhost.localdomain:46159}, state=RUNNABLE, group=TGRP-TestHdfsCloudBackupRestore]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)         at java.lang.Thread.run(Thread.java:748)   13) Thread[id=5880, name=IPC Server idle connection scanner for port 45315, state=WAITING, group=TGRP-TestHdfsCloudBackupRestore]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at java.util.TimerThread.mainLoop(Timer.java:526)         at java.util.TimerThread.run(Timer.java:505)   14) Thread[id=5864, name=qtp1033731312-5864, state=RUNNABLE, group=TGRP-TestHdfsCloudBackupRestore]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$58/1990976794.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)         at java.lang.Thread.run(Thread.java:748)   15) Thread[id=5862, name=pool-3-thread-1, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 15 threads leaked from SUITE scope at org.apache.solr.cloud.api.collections.TestHdfsCloudBackupRestore: 
   1) Thread[id=5871, name=Session-HouseKeeper-2eb24c26, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=5867, name=qtp1033731312-5867, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=5860, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]
        at java.lang.Object.wait(Native Method)
        at java.util.TimerThread.mainLoop(Timer.java:552)
        at java.util.TimerThread.run(Timer.java:505)
   4) Thread[id=5877, name=FSEditLogAsync, state=WAITING, group=TGRP-TestHdfsCloudBackupRestore]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)
        at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.dequeueEdit(FSEditLogAsync.java:221)
        at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.run(FSEditLogAsync.java:229)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=5866, name=qtp1033731312-5866, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=5869, name=qtp1033731312-5869, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   7) Thread[id=5870, name=qtp1033731312-5870, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   8) Thread[id=5868, name=qtp1033731312-5868, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   9) Thread[id=5879, name=Socket Reader #1 for port 45315, state=RUNNABLE, group=TGRP-TestHdfsCloudBackupRestore]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097)
        at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076)
  10) Thread[id=5861, name=org.apache.hadoop.util.JvmPauseMonitor$Monitor@1dcd311d, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192)
        at java.lang.Thread.run(Thread.java:748)
  11) Thread[id=5863, name=qtp1033731312-5863, state=RUNNABLE, group=TGRP-TestHdfsCloudBackupRestore]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$58/1990976794.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
        at java.lang.Thread.run(Thread.java:748)
  12) Thread[id=5865, name=qtp1033731312-5865-acceptor-0@57366549-ServerConnector@7e301298{HTTP/1.1,[http/1.1]}{localhost.localdomain:46159}, state=RUNNABLE, group=TGRP-TestHdfsCloudBackupRestore]
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
        at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385)
        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
        at java.lang.Thread.run(Thread.java:748)
  13) Thread[id=5880, name=IPC Server idle connection scanner for port 45315, state=WAITING, group=TGRP-TestHdfsCloudBackupRestore]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at java.util.TimerThread.mainLoop(Timer.java:526)
        at java.util.TimerThread.run(Timer.java:505)
  14) Thread[id=5864, name=qtp1033731312-5864, state=RUNNABLE, group=TGRP-TestHdfsCloudBackupRestore]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$58/1990976794.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
        at java.lang.Thread.run(Thread.java:748)
  15) Thread[id=5862, name=pool-3-thread-1, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([AA80BF4650F470FD]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.api.collections.TestHdfsCloudBackupRestore

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=5871, name=Session-HouseKeeper-2eb24c26, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=5867, name=qtp1033731312-5867, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=5860, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]         at java.lang.Object.wait(Native Method)         at java.util.TimerThread.mainLoop(Timer.java:552)         at java.util.TimerThread.run(Timer.java:505)    4) Thread[id=5866, name=qtp1033731312-5866, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=5869, name=qtp1033731312-5869, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=5870, name=qtp1033731312-5870, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    7) Thread[id=5868, name=qtp1033731312-5868, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    8) Thread[id=5879, name=Socket Reader #1 for port 45315, state=RUNNABLE, group=TGRP-TestHdfsCloudBackupRestore]         at sun.nio.ch.IOUtil.drain(Native Method)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097)         at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076)    9) Thread[id=5863, name=qtp1033731312-5863, state=RUNNABLE, group=TGRP-TestHdfsCloudBackupRestore]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$58/1990976794.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)         at java.lang.Thread.run(Thread.java:748)   10) Thread[id=5865, name=qtp1033731312-5865, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)   11) Thread[id=5880, name=IPC Server idle connection scanner for port 45315, state=WAITING, group=TGRP-TestHdfsCloudBackupRestore]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at java.util.TimerThread.mainLoop(Timer.java:526)         at java.util.TimerThread.run(Timer.java:505)   12) Thread[id=5864, name=qtp1033731312-5864, state=RUNNABLE, group=TGRP-TestHdfsCloudBackupRestore]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$58/1990976794.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.ja

[...truncated too long message...]

4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    5) Thread[id=24656, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-CheckHdfsIndexTest]
   [junit4]    >         at java.lang.Object.wait(Native Method)
   [junit4]    >         at java.util.TimerThread.mainLoop(Timer.java:552)
   [junit4]    >         at java.util.TimerThread.run(Timer.java:505)
   [junit4]    >    6) Thread[id=24666, name=qtp1278458977-24666, state=TIMED_WAITING, group=TGRP-CheckHdfsIndexTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    7) Thread[id=24661, name=qtp1278458977-24661, state=TIMED_WAITING, group=TGRP-CheckHdfsIndexTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    8) Thread[id=24658, name=pool-3-thread-1, state=TIMED_WAITING, group=TGRP-CheckHdfsIndexTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    9) Thread[id=24660, name=qtp1278458977-24660, state=RUNNABLE, group=TGRP-CheckHdfsIndexTest]
   [junit4]    >         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
   [junit4]    >         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
   [junit4]    >         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
   [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$$Lambda$59/1986404630.run(Unknown Source)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   10) Thread[id=24675, name=Socket Reader #1 for port 38425, state=RUNNABLE, group=TGRP-CheckHdfsIndexTest]
   [junit4]    >         at sun.nio.ch.IOUtil.drain(Native Method)
   [junit4]    >         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104)
   [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
   [junit4]    >         at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097)
   [junit4]    >         at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076)
   [junit4]    >   11) Thread[id=24659, name=qtp1278458977-24659, state=RUNNABLE, group=TGRP-CheckHdfsIndexTest]
   [junit4]    >         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
   [junit4]    >         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
   [junit4]    >         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
   [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$$Lambda$59/1986404630.run(Unknown Source)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   12) Thread[id=24664, name=qtp1278458977-24664, state=TIMED_WAITING, group=TGRP-CheckHdfsIndexTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   13) Thread[id=24663, name=qtp1278458977-24663, state=TIMED_WAITING, group=TGRP-CheckHdfsIndexTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([AA80BF4650F470FD]:0)
   [junit4] Completed [805/887 (3!)] on J2 in 14.97s, 0 tests, 3 errors <<< FAILURES!

[...truncated 9771 lines...]
BUILD FAILED
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/build.xml:652: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/build.xml:586: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid30526.hprof

Total time: 206 minutes 7 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any

[JENKINS] Lucene-Solr-NightlyTests-8.x - Build # 291 - Still Unstable

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-8.x/291/

9 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation

Error Message:
java.security.AccessControlException: access denied ("java.io.FilePermission" "<<ALL FILES>>" "execute")

Stack Trace:
com.google.common.util.concurrent.UncheckedExecutionException: java.security.AccessControlException: access denied ("java.io.FilePermission" "<<ALL FILES>>" "execute")
	at __randomizedtesting.SeedInfo.seed([C1EFB055BC51581D]:0)
	at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2050)
	at com.google.common.cache.LocalCache.get(LocalCache.java:3951)
	at com.google.common.cache.LocalCache.getOrLoad(LocalCache.java:3973)
	at com.google.common.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4957)
	at org.apache.hadoop.security.Groups.getGroups(Groups.java:228)
	at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation.getUsersFirstGroup(TestSolrCloudWithSecureImpersonation.java:64)
	at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation.getImpersonatorSettings(TestSolrCloudWithSecureImpersonation.java:87)
	at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation.startup(TestSolrCloudWithSecureImpersonation.java:102)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:878)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.security.AccessControlException: access denied ("java.io.FilePermission" "<<ALL FILES>>" "execute")
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.security.AccessController.checkPermission(AccessController.java:884)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkExec(SecurityManager.java:799)
	at java.lang.ProcessBuilder.start(ProcessBuilder.java:1018)
	at org.apache.hadoop.util.Shell.runCommand(Shell.java:938)
	at org.apache.hadoop.util.Shell.run(Shell.java:901)
	at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1213)
	at org.apache.hadoop.security.ShellBasedUnixGroupsMapping.getUnixGroups(ShellBasedUnixGroupsMapping.java:200)
	at org.apache.hadoop.security.ShellBasedUnixGroupsMapping.getGroups(ShellBasedUnixGroupsMapping.java:97)
	at org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback.getGroups(JniBasedUnixGroupsMappingWithFallback.java:51)
	at org.apache.hadoop.security.Groups$GroupCacheLoader.fetchGroupList(Groups.java:387)
	at org.apache.hadoop.security.Groups$GroupCacheLoader.load(Groups.java:321)
	at org.apache.hadoop.security.Groups$GroupCacheLoader.load(Groups.java:270)
	at com.google.common.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3527)
	at com.google.common.cache.LocalCache$Segment.loadSync(LocalCache.java:2276)
	at com.google.common.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2154)
	at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2044)
	... 31 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation

Error Message:
java.security.AccessControlException: access denied ("java.io.FilePermission" "<<ALL FILES>>" "execute")

Stack Trace:
com.google.common.util.concurrent.UncheckedExecutionException: java.security.AccessControlException: access denied ("java.io.FilePermission" "<<ALL FILES>>" "execute")
	at __randomizedtesting.SeedInfo.seed([C1EFB055BC51581D]:0)
	at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2050)
	at com.google.common.cache.LocalCache.get(LocalCache.java:3951)
	at com.google.common.cache.LocalCache.getOrLoad(LocalCache.java:3973)
	at com.google.common.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4957)
	at org.apache.hadoop.security.Groups.getGroups(Groups.java:228)
	at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation.getUsersFirstGroup(TestSolrCloudWithSecureImpersonation.java:64)
	at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation.getImpersonatorSettings(TestSolrCloudWithSecureImpersonation.java:87)
	at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation.shutdown(TestSolrCloudWithSecureImpersonation.java:162)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:901)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.security.AccessControlException: access denied ("java.io.FilePermission" "<<ALL FILES>>" "execute")
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.security.AccessController.checkPermission(AccessController.java:884)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkExec(SecurityManager.java:799)
	at java.lang.ProcessBuilder.start(ProcessBuilder.java:1018)
	at org.apache.hadoop.util.Shell.runCommand(Shell.java:938)
	at org.apache.hadoop.util.Shell.run(Shell.java:901)
	at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1213)
	at org.apache.hadoop.security.ShellBasedUnixGroupsMapping.getUnixGroups(ShellBasedUnixGroupsMapping.java:200)
	at org.apache.hadoop.security.ShellBasedUnixGroupsMapping.getGroups(ShellBasedUnixGroupsMapping.java:97)
	at org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback.getGroups(JniBasedUnixGroupsMappingWithFallback.java:51)
	at org.apache.hadoop.security.Groups$GroupCacheLoader.fetchGroupList(Groups.java:387)
	at org.apache.hadoop.security.Groups$GroupCacheLoader.load(Groups.java:321)
	at org.apache.hadoop.security.Groups$GroupCacheLoader.load(Groups.java:270)
	at com.google.common.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3527)
	at com.google.common.cache.LocalCache$Segment.loadSync(LocalCache.java:2276)
	at com.google.common.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2154)
	at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2044)
	... 30 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.autoscaling.SearchRateTriggerIntegrationTest

Error Message:
Clean up static fields (in @AfterClass?) and null them, your test still has references to classes of which the sizes cannot be measured due to security restrictions or Java 9 module encapsulation:   - private static org.apache.solr.client.solrj.cloud.SolrCloudManager org.apache.solr.cloud.autoscaling.SearchRateTriggerIntegrationTest.cloudManager

Stack Trace:
junit.framework.AssertionFailedError: Clean up static fields (in @AfterClass?) and null them, your test still has references to classes of which the sizes cannot be measured due to security restrictions or Java 9 module encapsulation:
  - private static org.apache.solr.client.solrj.cloud.SolrCloudManager org.apache.solr.cloud.autoscaling.SearchRateTriggerIntegrationTest.cloudManager
	at __randomizedtesting.SeedInfo.seed([C1EFB055BC51581D]:0)
	at com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule$1.afterAlways(StaticFieldsInvariantRule.java:146)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.security.AccessControlException: access denied ("java.lang.RuntimePermission" "accessClassInPackage.sun.nio.fs")
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.security.AccessController.checkPermission(AccessController.java:884)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkPackageAccess(SecurityManager.java:1564)
	at java.lang.Class.checkPackageAccess(Class.java:2372)
	at java.lang.Class.checkMemberAccess(Class.java:2351)
	at java.lang.Class.getDeclaredFields(Class.java:1915)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator$2.run(RamUsageEstimator.java:585)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator$2.run(RamUsageEstimator.java:582)
	at java.security.AccessController.doPrivileged(Native Method)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.createCacheEntry(RamUsageEstimator.java:582)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.measureSizeOf(RamUsageEstimator.java:545)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.sizeOfAll(RamUsageEstimator.java:387)
	at com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule$1.afterAlways(StaticFieldsInvariantRule.java:129)
	... 10 more


FAILED:  junit.framework.TestSuite.org.apache.solr.core.HdfsDirectoryFactoryTest

Error Message:
access denied ("java.io.FilePermission" "<<ALL FILES>>" "execute")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "<<ALL FILES>>" "execute")
	at __randomizedtesting.SeedInfo.seed([C1EFB055BC51581D]:0)
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.security.AccessController.checkPermission(AccessController.java:884)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkExec(SecurityManager.java:799)
	at java.lang.ProcessBuilder.start(ProcessBuilder.java:1018)
	at org.apache.hadoop.util.Shell.runCommand(Shell.java:938)
	at org.apache.hadoop.util.Shell.run(Shell.java:901)
	at org.apache.hadoop.fs.DF.getFilesystem(DF.java:74)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker$CheckedVolume.<init>(NameNodeResourceChecker.java:70)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker.addDirToCheck(NameNodeResourceChecker.java:166)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker.<init>(NameNodeResourceChecker.java:135)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startCommonServices(FSNamesystem.java:1168)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.startCommonServices(NameNode.java:791)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.initialize(NameNode.java:717)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.<init>(NameNode.java:940)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.<init>(NameNode.java:913)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.createNameNode(NameNode.java:1646)
	at org.apache.hadoop.hdfs.MiniDFSCluster.createNameNode(MiniDFSCluster.java:1314)
	at org.apache.hadoop.hdfs.MiniDFSCluster.configureNameService(MiniDFSCluster.java:1083)
	at org.apache.hadoop.hdfs.MiniDFSCluster.createNameNodesAndSetConf(MiniDFSCluster.java:958)
	at org.apache.hadoop.hdfs.MiniDFSCluster.initMiniDFSCluster(MiniDFSCluster.java:890)
	at org.apache.hadoop.hdfs.MiniDFSCluster.<init>(MiniDFSCluster.java:518)
	at org.apache.hadoop.hdfs.MiniDFSCluster$Builder.build(MiniDFSCluster.java:477)
	at org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:134)
	at org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:70)
	at org.apache.solr.core.HdfsDirectoryFactoryTest.setupClass(HdfsDirectoryFactoryTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:878)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.core.HdfsDirectoryFactoryTest

Error Message:
15 threads leaked from SUITE scope at org.apache.solr.core.HdfsDirectoryFactoryTest:     1) Thread[id=8129, name=pool-5-thread-1, state=TIMED_WAITING, group=TGRP-HdfsDirectoryFactoryTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=8134, name=qtp1924729069-8134, state=TIMED_WAITING, group=TGRP-HdfsDirectoryFactoryTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=8146, name=Socket Reader #1 for port 40309, state=RUNNABLE, group=TGRP-HdfsDirectoryFactoryTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097)         at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076)    4) Thread[id=8147, name=IPC Server idle connection scanner for port 40309, state=WAITING, group=TGRP-HdfsDirectoryFactoryTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at java.util.TimerThread.mainLoop(Timer.java:526)         at java.util.TimerThread.run(Timer.java:505)    5) Thread[id=8136, name=qtp1924729069-8136, state=TIMED_WAITING, group=TGRP-HdfsDirectoryFactoryTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=8137, name=qtp1924729069-8137, state=TIMED_WAITING, group=TGRP-HdfsDirectoryFactoryTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    7) Thread[id=8133, name=qtp1924729069-8133-acceptor-0@63b4aa5f-ServerConnector@22a6920{HTTP/1.1,[http/1.1]}{localhost.localdomain:41071}, state=RUNNABLE, group=TGRP-HdfsDirectoryFactoryTest]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)         at java.lang.Thread.run(Thread.java:748)    8) Thread[id=8128, name=org.apache.hadoop.util.JvmPauseMonitor$Monitor@e4d7b20, state=TIMED_WAITING, group=TGRP-HdfsDirectoryFactoryTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192)         at java.lang.Thread.run(Thread.java:748)    9) Thread[id=8130, name=qtp1924729069-8130, state=RUNNABLE, group=TGRP-HdfsDirectoryFactoryTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$69/1126322705.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)         at java.lang.Thread.run(Thread.java:748)   10) Thread[id=8131, name=qtp1924729069-8131, state=RUNNABLE, group=TGRP-HdfsDirectoryFactoryTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$69/1126322705.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)         at java.lang.Thread.run(Thread.java:748)   11) Thread[id=8138, name=Session-HouseKeeper-52cecf79, state=TIMED_WAITING, group=TGRP-HdfsDirectoryFactoryTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   12) Thread[id=8144, name=FSEditLogAsync, state=WAITING, group=TGRP-HdfsDirectoryFactoryTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)         at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.dequeueEdit(FSEditLogAsync.java:221)         at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.run(FSEditLogAsync.java:229)         at java.lang.Thread.run(Thread.java:748)   13) Thread[id=8132, name=qtp1924729069-8132, state=TIMED_WAITING, group=TGRP-HdfsDirectoryFactoryTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)   14) Thread[id=8127, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-HdfsDirectoryFactoryTest]         at java.lang.Object.wait(Native Method)         at java.util.TimerThread.mainLoop(Timer.java:552)         at java.util.TimerThread.run(Timer.java:505)   15) Thread[id=8135, name=qtp1924729069-8135, state=TIMED_WAITING, group=TGRP-HdfsDirectoryFactoryTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 15 threads leaked from SUITE scope at org.apache.solr.core.HdfsDirectoryFactoryTest: 
   1) Thread[id=8129, name=pool-5-thread-1, state=TIMED_WAITING, group=TGRP-HdfsDirectoryFactoryTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=8134, name=qtp1924729069-8134, state=TIMED_WAITING, group=TGRP-HdfsDirectoryFactoryTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=8146, name=Socket Reader #1 for port 40309, state=RUNNABLE, group=TGRP-HdfsDirectoryFactoryTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097)
        at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076)
   4) Thread[id=8147, name=IPC Server idle connection scanner for port 40309, state=WAITING, group=TGRP-HdfsDirectoryFactoryTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at java.util.TimerThread.mainLoop(Timer.java:526)
        at java.util.TimerThread.run(Timer.java:505)
   5) Thread[id=8136, name=qtp1924729069-8136, state=TIMED_WAITING, group=TGRP-HdfsDirectoryFactoryTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=8137, name=qtp1924729069-8137, state=TIMED_WAITING, group=TGRP-HdfsDirectoryFactoryTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   7) Thread[id=8133, name=qtp1924729069-8133-acceptor-0@63b4aa5f-ServerConnector@22a6920{HTTP/1.1,[http/1.1]}{localhost.localdomain:41071}, state=RUNNABLE, group=TGRP-HdfsDirectoryFactoryTest]
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
        at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385)
        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
        at java.lang.Thread.run(Thread.java:748)
   8) Thread[id=8128, name=org.apache.hadoop.util.JvmPauseMonitor$Monitor@e4d7b20, state=TIMED_WAITING, group=TGRP-HdfsDirectoryFactoryTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192)
        at java.lang.Thread.run(Thread.java:748)
   9) Thread[id=8130, name=qtp1924729069-8130, state=RUNNABLE, group=TGRP-HdfsDirectoryFactoryTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$69/1126322705.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
        at java.lang.Thread.run(Thread.java:748)
  10) Thread[id=8131, name=qtp1924729069-8131, state=RUNNABLE, group=TGRP-HdfsDirectoryFactoryTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$69/1126322705.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
        at java.lang.Thread.run(Thread.java:748)
  11) Thread[id=8138, name=Session-HouseKeeper-52cecf79, state=TIMED_WAITING, group=TGRP-HdfsDirectoryFactoryTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  12) Thread[id=8144, name=FSEditLogAsync, state=WAITING, group=TGRP-HdfsDirectoryFactoryTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)
        at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.dequeueEdit(FSEditLogAsync.java:221)
        at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.run(FSEditLogAsync.java:229)
        at java.lang.Thread.run(Thread.java:748)
  13) Thread[id=8132, name=qtp1924729069-8132, state=TIMED_WAITING, group=TGRP-HdfsDirectoryFactoryTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
  14) Thread[id=8127, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-HdfsDirectoryFactoryTest]
        at java.lang.Object.wait(Native Method)
        at java.util.TimerThread.mainLoop(Timer.java:552)
        at java.util.TimerThread.run(Timer.java:505)
  15) Thread[id=8135, name=qtp1924729069-8135, state=TIMED_WAITING, group=TGRP-HdfsDirectoryFactoryTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([C1EFB055BC51581D]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.core.HdfsDirectoryFactoryTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=8129, name=pool-5-thread-1, state=TIMED_WAITING, group=TGRP-HdfsDirectoryFactoryTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=8134, name=qtp1924729069-8134, state=TIMED_WAITING, group=TGRP-HdfsDirectoryFactoryTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=8146, name=Socket Reader #1 for port 40309, state=RUNNABLE, group=TGRP-HdfsDirectoryFactoryTest]         at sun.nio.ch.IOUtil.drain(Native Method)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097)         at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076)    4) Thread[id=8147, name=IPC Server idle connection scanner for port 40309, state=WAITING, group=TGRP-HdfsDirectoryFactoryTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at java.util.TimerThread.mainLoop(Timer.java:526)         at java.util.TimerThread.run(Timer.java:505)    5) Thread[id=8136, name=qtp1924729069-8136, state=TIMED_WAITING, group=TGRP-HdfsDirectoryFactoryTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=8137, name=qtp1924729069-8137, state=TIMED_WAITING, group=TGRP-HdfsDirectoryFactoryTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    7) Thread[id=8133, name=qtp1924729069-8133, state=TIMED_WAITING, group=TGRP-HdfsDirectoryFactoryTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    8) Thread[id=8130, name=qtp1924729069-8130, state=RUNNABLE, group=TGRP-HdfsDirectoryFactoryTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$69/1126322705.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)         at java.lang.Thread.run(Thread.java:748)    9) Thread[id=8131, name=qtp1924729069-8131, state=RUNNABLE, group=TGRP-HdfsDirectoryFactoryTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$69/1126322705.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)         at java.lang.Thread.run(Thread.java:748)   10) Thread[id=8138, name=Session-HouseKeeper-52cecf79, state=TIMED_WAITING, group=TGRP-HdfsDirectoryFactoryTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   11) Thread[id=8132, name=qtp1924729069-8132, state=TIMED_WAITING, group=TGRP-HdfsDirectoryFactoryTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)   12) Thread[id=8127, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-HdfsDirectoryFactoryTest]         at java.lang.Object.wait(Native Method)         at java.util.TimerThread.mainLoop(Timer.java:552)         at java.util.TimerThread.run(Timer.java:505)   13) Thread[id=8135, name=qtp1924729069-8135, state=TIMED_WAITING, group=TGRP-HdfsDirectoryFactoryTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=8129, name=pool-5-thread-1, state=TIMED_WAITING, group=TGRP-HdfsDirectoryFactoryTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=8134, name=qtp1924729069-8134, state=TIMED_WAITING, group=TGRP-HdfsDirectoryFactoryTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=8146, name=Socket Reader #1 for port 40309, state=RUNNABLE, group=TGRP-HdfsDirectoryFactoryTest]
        at sun.nio.ch.IOUtil.drain(Native Method)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097)
        at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076)
   4) Thread[id=8147, name=IPC Server idle connection scanner for port 40309, state=WAITING, group=TGRP-HdfsDirectoryFactoryTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at java.util.TimerThread.mainLoop(Timer.java:526)
        at java.util.TimerThread.run(Timer.java:505)
   5) Thread[id=8136, name=qtp1924729069-8136, state=TIMED_WAITING, group=TGRP-HdfsDirectoryFactoryTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=8137, name=qtp1924729069-8137, state=TIMED_WAITING, group=TGRP-HdfsDirectoryFactoryTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   7) Thread[id=8133, name=qtp1924729069-8133, state=TIMED_WAITING, group=TGRP-HdfsDirectoryFactoryTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   8) Thread[id=8130, name=qtp1924729069-8130, state=RUNNABLE, group=TGRP-HdfsDirectoryFactoryTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$69/1126322705.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
        at java.lang.Thread.run(Thread.java:748)
   9) Thread[id=8131, name=qtp1924729069-8131, state=RUNNABLE, group=TGRP-HdfsDirectoryFactoryTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$69/1126322705.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
        at java.lang.Thread.run(Thread.java:748)
  10) Thread[id=8138, name=Session-HouseKeeper-52cecf79, state=TIMED_WAITING, group=TGRP-HdfsDirectoryFactoryTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  11) Thread[id=8132, name=qtp1924729069-8132, state=TIMED_WAITING, group=TGRP-HdfsDirectoryFactoryTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
  12) Thread[id=8127, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-HdfsDirectoryFactoryTest]
        at java.lang.Object.wait(Native Method)
        at java.util.TimerThread.mainLoop(Timer.java:552)
        at java.util.TimerThread.run(Timer.java:505)
  13) Thread[id=8135, name=qtp1924729069-8135, state=TIMED_WAITING, group=TGRP-HdfsDirectoryFactoryTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([C1EFB055BC51581D]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.store.hdfs.HdfsLockFactoryTest

Error Message:
access denied ("java.io.FilePermission" "<<ALL FILES>>" "execute")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "<<ALL FILES>>" "execute")
	at __randomizedtesting.SeedInfo.seed([C1EFB055BC51581D]:0)
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.security.AccessController.checkPermission(AccessController.java:884)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkExec(SecurityManager.java:799)
	at java.lang.ProcessBuilder.start(ProcessBuilder.java:1018)
	at org.apache.hadoop.util.Shell.runCommand(Shell.java:938)
	at org.apache.hadoop.util.Shell.run(Shell.java:901)
	at org.apache.hadoop.fs.DF.getFilesystem(DF.java:74)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker$CheckedVolume.<init>(NameNodeResourceChecker.java:70)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker.addDirToCheck(NameNodeResourceChecker.java:166)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker.<init>(NameNodeResourceChecker.java:135)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startCommonServices(FSNamesystem.java:1168)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.startCommonServices(NameNode.java:791)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.initialize(NameNode.java:717)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.<init>(NameNode.java:940)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.<init>(NameNode.java:913)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.createNameNode(NameNode.java:1646)
	at org.apache.hadoop.hdfs.MiniDFSCluster.createNameNode(MiniDFSCluster.java:1314)
	at org.apache.hadoop.hdfs.MiniDFSCluster.configureNameService(MiniDFSCluster.java:1083)
	at org.apache.hadoop.hdfs.MiniDFSCluster.createNameNodesAndSetConf(MiniDFSCluster.java:958)
	at org.apache.hadoop.hdfs.MiniDFSCluster.initMiniDFSCluster(MiniDFSCluster.java:890)
	at org.apache.hadoop.hdfs.MiniDFSCluster.<init>(MiniDFSCluster.java:518)
	at org.apache.hadoop.hdfs.MiniDFSCluster$Builder.build(MiniDFSCluster.java:477)
	at org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:134)
	at org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:66)
	at org.apache.solr.store.hdfs.HdfsLockFactoryTest.beforeClass(HdfsLockFactoryTest.java:46)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:878)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.store.hdfs.HdfsLockFactoryTest

Error Message:
15 threads leaked from SUITE scope at org.apache.solr.store.hdfs.HdfsLockFactoryTest:     1) Thread[id=7861, name=FSEditLogAsync, state=WAITING, group=TGRP-HdfsLockFactoryTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)         at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.dequeueEdit(FSEditLogAsync.java:221)         at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.run(FSEditLogAsync.java:229)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=7844, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-HdfsLockFactoryTest]         at java.lang.Object.wait(Native Method)         at java.util.TimerThread.mainLoop(Timer.java:552)         at java.util.TimerThread.run(Timer.java:505)    3) Thread[id=7849, name=qtp740740904-7849-acceptor-0@1fb9ed08-ServerConnector@52d506ec{HTTP/1.1,[http/1.1]}{localhost.localdomain:45839}, state=RUNNABLE, group=TGRP-HdfsLockFactoryTest]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=7852, name=qtp740740904-7852, state=TIMED_WAITING, group=TGRP-HdfsLockFactoryTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=7851, name=qtp740740904-7851, state=TIMED_WAITING, group=TGRP-HdfsLockFactoryTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=7850, name=qtp740740904-7850, state=TIMED_WAITING, group=TGRP-HdfsLockFactoryTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    7) Thread[id=7864, name=IPC Server idle connection scanner for port 46647, state=WAITING, group=TGRP-HdfsLockFactoryTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at java.util.TimerThread.mainLoop(Timer.java:526)         at java.util.TimerThread.run(Timer.java:505)    8) Thread[id=7846, name=pool-16-thread-1, state=TIMED_WAITING, group=TGRP-HdfsLockFactoryTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    9) Thread[id=7854, name=qtp740740904-7854, state=TIMED_WAITING, group=TGRP-HdfsLockFactoryTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)   10) Thread[id=7845, name=org.apache.hadoop.util.JvmPauseMonitor$Monitor@1181d932, state=TIMED_WAITING, group=TGRP-HdfsLockFactoryTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192)         at java.lang.Thread.run(Thread.java:748)   11) Thread[id=7848, name=qtp740740904-7848, state=RUNNABLE, group=TGRP-HdfsLockFactoryTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$47/246257083.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)         at java.lang.Thread.run(Thread.java:748)   12) Thread[id=7863, name=Socket Reader #1 for port 46647, state=RUNNABLE, group=TGRP-HdfsLockFactoryTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097)         at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076)   13) Thread[id=7847, name=qtp740740904-7847, state=RUNNABLE, group=TGRP-HdfsLockFactoryTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$47/246257083.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)         at java.lang.Thread.run(Thread.java:748)   14) Thread[id=7853, name=qtp740740904-7853, state=TIMED_WAITING, group=TGRP-HdfsLockFactoryTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)   15) Thread[id=7855, name=Session-HouseKeeper-26e1ecc2, state=TIMED_WAITING, group=TGRP-HdfsLockFactoryTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 15 threads leaked from SUITE scope at org.apache.solr.store.hdfs.HdfsLockFactoryTest: 
   1) Thread[id=7861, name=FSEditLogAsync, state=WAITING, group=TGRP-HdfsLockFactoryTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)
        at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.dequeueEdit(FSEditLogAsync.java:221)
        at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.run(FSEditLogAsync.java:229)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=7844, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-HdfsLockFactoryTest]
        at java.lang.Object.wait(Native Method)
        at java.util.TimerThread.mainLoop(Timer.java:552)
        at java.util.TimerThread.run(Timer.java:505)
   3) Thread[id=7849, name=qtp740740904-7849-acceptor-0@1fb9ed08-ServerConnector@52d506ec{HTTP/1.1,[http/1.1]}{localhost.localdomain:45839}, state=RUNNABLE, group=TGRP-HdfsLockFactoryTest]
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
        at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385)
        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=7852, name=qtp740740904-7852, state=TIMED_WAITING, group=TGRP-HdfsLockFactoryTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=7851, name=qtp740740904-7851, state=TIMED_WAITING, group=TGRP-HdfsLockFactoryTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=7850, name=qtp740740904-7850, state=TIMED_WAITING, group=TGRP-HdfsLockFactoryTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   7) Thread[id=7864, name=IPC Server idle connection scanner for port 46647, state=WAITING, group=TGRP-HdfsLockFactoryTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at java.util.TimerThread.mainLoop(Timer.java:526)
        at java.util.TimerThread.run(Timer.java:505)
   8) Thread[id=7846, name=pool-16-thread-1, state=TIMED_WAITING, group=TGRP-HdfsLockFactoryTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   9) Thread[id=7854, name=qtp740740904-7854, state=TIMED_WAITING, group=TGRP-HdfsLockFactoryTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
  10) Thread[id=7845, name=org.apache.hadoop.util.JvmPauseMonitor$Monitor@1181d932, state=TIMED_WAITING, group=TGRP-HdfsLockFactoryTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192)
        at java.lang.Thread.run(Thread.java:748)
  11) Thread[id=7848, name=qtp740740904-7848, state=RUNNABLE, group=TGRP-HdfsLockFactoryTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$47/246257083.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
        at java.lang.Thread.run(Thread.java:748)
  12) Thread[id=7863, name=Socket Reader #1 for port 46647, state=RUNNABLE, group=TGRP-HdfsLockFactoryTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097)
        at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076)
  13) Thread[id=7847, name=qtp740740904-7847, state=RUNNABLE, group=TGRP-HdfsLockFactoryTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$47/246257083.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
        at java.lang.Thread.run(Thread.java:748)
  14) Thread[id=7853, name=qtp740740904-7853, state=TIMED_WAITING, group=TGRP-HdfsLockFactoryTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
  15) Thread[id=7855, name=Session-HouseKeeper-26e1ecc2, state=TIMED_WAITING, group=TGRP-HdfsLockFactoryTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([C1EFB055BC51581D]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.store.hdfs.HdfsLockFactoryTest

Error Message:
There are still zombie threads 

[...truncated too long message...]

junit4]   2> 1158119 WARN  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[C1EFB055BC51581D]-worker) [     ] o.a.h.u.NativeCodeLoader Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
   [junit4]   2> 1158193 INFO  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[C1EFB055BC51581D]-worker) [     ] o.a.s.SolrTestCaseJ4 ------------------------------------------------------- Done waiting for tracked resources to be released
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene84): {}, docValues:{}, maxPointsInLeafNode=455, maxMBSortInHeap=6.448337811605283, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@4a2f9498), locale=es-DO, timezone=US/Mountain
   [junit4]   2> NOTE: Linux 4.15.0-54-generic amd64/Oracle Corporation 1.8.0_191 (64-bit)/cpus=4,threads=1,free=192125848,total=527433728
   [junit4]   2> NOTE: All tests run in this JVM: [TestSystemIdResolver, PackageManagerCLITest, TestPerFieldSimilarity, HLLSerializationTest, TestNumericTokenStream, TestStressReorder, TestAuthenticationFramework, DirectSolrSpellCheckerTest, TestSolrCachePerf, SolrJmxReporterCloudTest, ShardRoutingTest, CircularListTest, SuggesterFSTTest, CachingDirectoryFactoryTest, SuggesterWFSTTest, TestSolrIndexConfig, TestUseDocValuesAsStored2, V2StandaloneTest, MultiDestinationAuditLoggerTest, CoreMergeIndexesAdminHandlerTest, BasicDistributedZk2Test, TemplateUpdateProcessorTest, SolrIndexMetricsTest, TestPseudoReturnFields, JavabinLoaderTest, TestDownShardTolerantSearch, RequestHandlersTest, TestConfigReload, TestDistribPackageStore, BlockJoinFacetDistribTest, TestBulkSchemaAPI, BlockCacheTest, SolrCLIZkUtilsTest, TestCloudManagedSchema, TestRandomFaceting, TestTolerantUpdateProcessorRandomCloud, TestSubQueryTransformer, TestSha256AuthenticationProvider, TestMultiValuedNumericRangeQuery, RuleEngineTest, SearchHandlerTest, ResponseHeaderTest, JWTAuthPluginTest, TestExceedMaxTermLength, TestDynamicFieldResource, SynonymTokenizerTest, TestOverriddenPrefixQueryForCustomFieldType, PeerSyncWithLeaderAndIndexFingerprintCachingTest, CloudExitableDirectoryReaderTest, DirectUpdateHandlerTest, AnalyticsQueryTest, ComputePlanActionTest, TestFieldCacheWithThreads, DistributedUpdateProcessorTest, TestRebalanceLeaders, PluginInfoTest, DirectoryFactoryTest, TestCloudSearcherWarming, TestPointFields, CopyFieldTest, CollectionTooManyReplicasTest, TestDeleteCollectionOnDownNodes, BlobRepositoryCloudTest, TestNumericTerms64, CreateCollectionCleanupTest, NodeLostTriggerIntegrationTest, CdcrBidirectionalTest, TestSimDistributedQueue, TestReplicationHandler, TestPrepRecovery, TestLegacyNumericRangeQueryBuilder, TestBadConfig, ForceLeaderWithTlogReplicasTest, LeaderFailoverAfterPartitionTest, TestSolrCloudWithHadoopAuthPlugin, QueryParsingTest, TestRawTransformer, CloneFieldUpdateProcessorFactoryTest, ZkStateReaderTest, TestManagedSchemaAPI, TestBinaryResponseWriter, TestSolrCloudWithSecureImpersonation]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestSolrCloudWithSecureImpersonation -Dtests.seed=C1EFB055BC51581D -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/test-data/enwiki.random.lines.txt -Dtests.locale=es-DO -Dtests.timezone=US/Mountain -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   0.00s J1 | TestSolrCloudWithSecureImpersonation (suite) <<<
   [junit4]    > Throwable #1: com.google.common.util.concurrent.UncheckedExecutionException: java.security.AccessControlException: access denied ("java.io.FilePermission" "<<ALL FILES>>" "execute")
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([C1EFB055BC51581D]:0)
   [junit4]    > 	at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2050)
   [junit4]    > 	at com.google.common.cache.LocalCache.get(LocalCache.java:3951)
   [junit4]    > 	at com.google.common.cache.LocalCache.getOrLoad(LocalCache.java:3973)
   [junit4]    > 	at com.google.common.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4957)
   [junit4]    > 	at org.apache.hadoop.security.Groups.getGroups(Groups.java:228)
   [junit4]    > 	at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation.getUsersFirstGroup(TestSolrCloudWithSecureImpersonation.java:64)
   [junit4]    > 	at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation.getImpersonatorSettings(TestSolrCloudWithSecureImpersonation.java:87)
   [junit4]    > 	at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation.startup(TestSolrCloudWithSecureImpersonation.java:102)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > Caused by: java.security.AccessControlException: access denied ("java.io.FilePermission" "<<ALL FILES>>" "execute")
   [junit4]    > 	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
   [junit4]    > 	at java.security.AccessController.checkPermission(AccessController.java:884)
   [junit4]    > 	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
   [junit4]    > 	at java.lang.SecurityManager.checkExec(SecurityManager.java:799)
   [junit4]    > 	at java.lang.ProcessBuilder.start(ProcessBuilder.java:1018)
   [junit4]    > 	at org.apache.hadoop.util.Shell.runCommand(Shell.java:938)
   [junit4]    > 	at org.apache.hadoop.util.Shell.run(Shell.java:901)
   [junit4]    > 	at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1213)
   [junit4]    > 	at org.apache.hadoop.security.ShellBasedUnixGroupsMapping.getUnixGroups(ShellBasedUnixGroupsMapping.java:200)
   [junit4]    > 	at org.apache.hadoop.security.ShellBasedUnixGroupsMapping.getGroups(ShellBasedUnixGroupsMapping.java:97)
   [junit4]    > 	at org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback.getGroups(JniBasedUnixGroupsMappingWithFallback.java:51)
   [junit4]    > 	at org.apache.hadoop.security.Groups$GroupCacheLoader.fetchGroupList(Groups.java:387)
   [junit4]    > 	at org.apache.hadoop.security.Groups$GroupCacheLoader.load(Groups.java:321)
   [junit4]    > 	at org.apache.hadoop.security.Groups$GroupCacheLoader.load(Groups.java:270)
   [junit4]    > 	at com.google.common.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3527)
   [junit4]    > 	at com.google.common.cache.LocalCache$Segment.loadSync(LocalCache.java:2276)
   [junit4]    > 	at com.google.common.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2154)
   [junit4]    > 	at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2044)
   [junit4]    > 	... 31 moreThrowable #2: com.google.common.util.concurrent.UncheckedExecutionException: java.security.AccessControlException: access denied ("java.io.FilePermission" "<<ALL FILES>>" "execute")
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([C1EFB055BC51581D]:0)
   [junit4]    > 	at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2050)
   [junit4]    > 	at com.google.common.cache.LocalCache.get(LocalCache.java:3951)
   [junit4]    > 	at com.google.common.cache.LocalCache.getOrLoad(LocalCache.java:3973)
   [junit4]    > 	at com.google.common.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4957)
   [junit4]    > 	at org.apache.hadoop.security.Groups.getGroups(Groups.java:228)
   [junit4]    > 	at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation.getUsersFirstGroup(TestSolrCloudWithSecureImpersonation.java:64)
   [junit4]    > 	at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation.getImpersonatorSettings(TestSolrCloudWithSecureImpersonation.java:87)
   [junit4]    > 	at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation.shutdown(TestSolrCloudWithSecureImpersonation.java:162)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > Caused by: java.security.AccessControlException: access denied ("java.io.FilePermission" "<<ALL FILES>>" "execute")
   [junit4]    > 	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
   [junit4]    > 	at java.security.AccessController.checkPermission(AccessController.java:884)
   [junit4]    > 	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
   [junit4]    > 	at java.lang.SecurityManager.checkExec(SecurityManager.java:799)
   [junit4]    > 	at java.lang.ProcessBuilder.start(ProcessBuilder.java:1018)
   [junit4]    > 	at org.apache.hadoop.util.Shell.runCommand(Shell.java:938)
   [junit4]    > 	at org.apache.hadoop.util.Shell.run(Shell.java:901)
   [junit4]    > 	at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1213)
   [junit4]    > 	at org.apache.hadoop.security.ShellBasedUnixGroupsMapping.getUnixGroups(ShellBasedUnixGroupsMapping.java:200)
   [junit4]    > 	at org.apache.hadoop.security.ShellBasedUnixGroupsMapping.getGroups(ShellBasedUnixGroupsMapping.java:97)
   [junit4]    > 	at org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback.getGroups(JniBasedUnixGroupsMappingWithFallback.java:51)
   [junit4]    > 	at org.apache.hadoop.security.Groups$GroupCacheLoader.fetchGroupList(Groups.java:387)
   [junit4]    > 	at org.apache.hadoop.security.Groups$GroupCacheLoader.load(Groups.java:321)
   [junit4]    > 	at org.apache.hadoop.security.Groups$GroupCacheLoader.load(Groups.java:270)
   [junit4]    > 	at com.google.common.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3527)
   [junit4]    > 	at com.google.common.cache.LocalCache$Segment.loadSync(LocalCache.java:2276)
   [junit4]    > 	at com.google.common.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2154)
   [junit4]    > 	at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2044)
   [junit4]    > 	... 30 more
   [junit4] Completed [824/887 (4!)] on J1 in 0.26s, 0 tests, 2 errors <<< FAILURES!

[...truncated 53149 lines...]

[JENKINS] Lucene-Solr-NightlyTests-8.x - Build # 290 - Still Unstable

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-8.x/290/

9 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.MoveReplicaHDFSFailoverTest

Error Message:
access denied ("java.io.FilePermission" "<<ALL FILES>>" "execute")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "<<ALL FILES>>" "execute")
	at __randomizedtesting.SeedInfo.seed([DDEF8FCDB3B9404B]:0)
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.security.AccessController.checkPermission(AccessController.java:884)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkExec(SecurityManager.java:799)
	at java.lang.ProcessBuilder.start(ProcessBuilder.java:1018)
	at org.apache.hadoop.util.Shell.runCommand(Shell.java:938)
	at org.apache.hadoop.util.Shell.run(Shell.java:901)
	at org.apache.hadoop.fs.DF.getFilesystem(DF.java:74)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker$CheckedVolume.<init>(NameNodeResourceChecker.java:70)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker.addDirToCheck(NameNodeResourceChecker.java:166)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker.<init>(NameNodeResourceChecker.java:135)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startCommonServices(FSNamesystem.java:1168)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.startCommonServices(NameNode.java:791)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.initialize(NameNode.java:717)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.<init>(NameNode.java:940)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.<init>(NameNode.java:913)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.createNameNode(NameNode.java:1646)
	at org.apache.hadoop.hdfs.MiniDFSCluster.createNameNode(MiniDFSCluster.java:1314)
	at org.apache.hadoop.hdfs.MiniDFSCluster.configureNameService(MiniDFSCluster.java:1083)
	at org.apache.hadoop.hdfs.MiniDFSCluster.createNameNodesAndSetConf(MiniDFSCluster.java:958)
	at org.apache.hadoop.hdfs.MiniDFSCluster.initMiniDFSCluster(MiniDFSCluster.java:890)
	at org.apache.hadoop.hdfs.MiniDFSCluster.<init>(MiniDFSCluster.java:518)
	at org.apache.hadoop.hdfs.MiniDFSCluster$Builder.build(MiniDFSCluster.java:477)
	at org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:134)
	at org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:66)
	at org.apache.solr.cloud.MoveReplicaHDFSFailoverTest.setupClass(MoveReplicaHDFSFailoverTest.java:55)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:878)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.MoveReplicaHDFSFailoverTest

Error Message:
15 threads leaked from SUITE scope at org.apache.solr.cloud.MoveReplicaHDFSFailoverTest:     1) Thread[id=1143, name=qtp315643927-1143, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=1139, name=qtp315643927-1139, state=RUNNABLE, group=TGRP-MoveReplicaHDFSFailoverTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$61/1270351414.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=1140, name=qtp315643927-1140, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=1153, name=Socket Reader #1 for port 39257, state=RUNNABLE, group=TGRP-MoveReplicaHDFSFailoverTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097)         at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076)    5) Thread[id=1134, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]         at java.lang.Object.wait(Native Method)         at java.util.TimerThread.mainLoop(Timer.java:552)         at java.util.TimerThread.run(Timer.java:505)    6) Thread[id=1144, name=qtp315643927-1144, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    7) Thread[id=1136, name=pool-3-thread-1, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    8) Thread[id=1142, name=qtp315643927-1142, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    9) Thread[id=1145, name=Session-HouseKeeper-3428b9ed, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   10) Thread[id=1137, name=qtp315643927-1137-acceptor-0@15ed6f93-ServerConnector@5361ec1e{HTTP/1.1,[http/1.1]}{lucene2-us-west.apache.org:40037}, state=RUNNABLE, group=TGRP-MoveReplicaHDFSFailoverTest]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)         at java.lang.Thread.run(Thread.java:748)   11) Thread[id=1138, name=qtp315643927-1138, state=RUNNABLE, group=TGRP-MoveReplicaHDFSFailoverTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$61/1270351414.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)         at java.lang.Thread.run(Thread.java:748)   12) Thread[id=1135, name=org.apache.hadoop.util.JvmPauseMonitor$Monitor@46c0e86d, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192)         at java.lang.Thread.run(Thread.java:748)   13) Thread[id=1151, name=FSEditLogAsync, state=WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)         at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.dequeueEdit(FSEditLogAsync.java:221)         at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.run(FSEditLogAsync.java:229)         at java.lang.Thread.run(Thread.java:748)   14) Thread[id=1154, name=IPC Server idle connection scanner for port 39257, state=WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at java.util.TimerThread.mainLoop(Timer.java:526)         at java.util.TimerThread.run(Timer.java:505)   15) Thread[id=1141, name=qtp315643927-1141, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 15 threads leaked from SUITE scope at org.apache.solr.cloud.MoveReplicaHDFSFailoverTest: 
   1) Thread[id=1143, name=qtp315643927-1143, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=1139, name=qtp315643927-1139, state=RUNNABLE, group=TGRP-MoveReplicaHDFSFailoverTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$61/1270351414.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=1140, name=qtp315643927-1140, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=1153, name=Socket Reader #1 for port 39257, state=RUNNABLE, group=TGRP-MoveReplicaHDFSFailoverTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097)
        at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076)
   5) Thread[id=1134, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]
        at java.lang.Object.wait(Native Method)
        at java.util.TimerThread.mainLoop(Timer.java:552)
        at java.util.TimerThread.run(Timer.java:505)
   6) Thread[id=1144, name=qtp315643927-1144, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   7) Thread[id=1136, name=pool-3-thread-1, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   8) Thread[id=1142, name=qtp315643927-1142, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   9) Thread[id=1145, name=Session-HouseKeeper-3428b9ed, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  10) Thread[id=1137, name=qtp315643927-1137-acceptor-0@15ed6f93-ServerConnector@5361ec1e{HTTP/1.1,[http/1.1]}{lucene2-us-west.apache.org:40037}, state=RUNNABLE, group=TGRP-MoveReplicaHDFSFailoverTest]
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
        at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385)
        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
        at java.lang.Thread.run(Thread.java:748)
  11) Thread[id=1138, name=qtp315643927-1138, state=RUNNABLE, group=TGRP-MoveReplicaHDFSFailoverTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$61/1270351414.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
        at java.lang.Thread.run(Thread.java:748)
  12) Thread[id=1135, name=org.apache.hadoop.util.JvmPauseMonitor$Monitor@46c0e86d, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192)
        at java.lang.Thread.run(Thread.java:748)
  13) Thread[id=1151, name=FSEditLogAsync, state=WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)
        at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.dequeueEdit(FSEditLogAsync.java:221)
        at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.run(FSEditLogAsync.java:229)
        at java.lang.Thread.run(Thread.java:748)
  14) Thread[id=1154, name=IPC Server idle connection scanner for port 39257, state=WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at java.util.TimerThread.mainLoop(Timer.java:526)
        at java.util.TimerThread.run(Timer.java:505)
  15) Thread[id=1141, name=qtp315643927-1141, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([DDEF8FCDB3B9404B]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.MoveReplicaHDFSFailoverTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=1143, name=qtp315643927-1143, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=1139, name=qtp315643927-1139, state=RUNNABLE, group=TGRP-MoveReplicaHDFSFailoverTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$61/1270351414.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=1140, name=qtp315643927-1140, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=1153, name=Socket Reader #1 for port 39257, state=RUNNABLE, group=TGRP-MoveReplicaHDFSFailoverTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097)         at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076)    5) Thread[id=1134, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]         at java.lang.Object.wait(Native Method)         at java.util.TimerThread.mainLoop(Timer.java:552)         at java.util.TimerThread.run(Timer.java:505)    6) Thread[id=1144, name=qtp315643927-1144, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    7) Thread[id=1136, name=pool-3-thread-1, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    8) Thread[id=1142, name=qtp315643927-1142, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    9) Thread[id=1145, name=Session-HouseKeeper-3428b9ed, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   10) Thread[id=1137, name=qtp315643927-1137, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)   11) Thread[id=1138, name=qtp315643927-1138, state=RUNNABLE, group=TGRP-MoveReplicaHDFSFailoverTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$61/1270351414.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)         at java.lang.Thread.run(Thread.java:748)   12) Thread[id=1154, name=IPC Server idle connection scanner for port 39257, state=WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at java.util.TimerThread.mainLoop(Timer.java:526)         at java.util.TimerThread.run(Timer.java:505)   13) Thread[id=1141, name=qtp315643927-1141, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=1143, name=qtp315643927-1143, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=1139, name=qtp315643927-1139, state=RUNNABLE, group=TGRP-MoveReplicaHDFSFailoverTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$61/1270351414.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=1140, name=qtp315643927-1140, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=1153, name=Socket Reader #1 for port 39257, state=RUNNABLE, group=TGRP-MoveReplicaHDFSFailoverTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097)
        at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076)
   5) Thread[id=1134, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]
        at java.lang.Object.wait(Native Method)
        at java.util.TimerThread.mainLoop(Timer.java:552)
        at java.util.TimerThread.run(Timer.java:505)
   6) Thread[id=1144, name=qtp315643927-1144, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   7) Thread[id=1136, name=pool-3-thread-1, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   8) Thread[id=1142, name=qtp315643927-1142, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   9) Thread[id=1145, name=Session-HouseKeeper-3428b9ed, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  10) Thread[id=1137, name=qtp315643927-1137, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
  11) Thread[id=1138, name=qtp315643927-1138, state=RUNNABLE, group=TGRP-MoveReplicaHDFSFailoverTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$61/1270351414.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
        at java.lang.Thread.run(Thread.java:748)
  12) Thread[id=1154, name=IPC Server idle connection scanner for port 39257, state=WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at java.util.TimerThread.mainLoop(Timer.java:526)
        at java.util.TimerThread.run(Timer.java:505)
  13) Thread[id=1141, name=qtp315643927-1141, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSFailoverTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([DDEF8FCDB3B9404B]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest

Error Message:
access denied ("java.io.FilePermission" "<<ALL FILES>>" "execute")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "<<ALL FILES>>" "execute")
	at __randomizedtesting.SeedInfo.seed([DDEF8FCDB3B9404B]:0)
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.security.AccessController.checkPermission(AccessController.java:884)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkExec(SecurityManager.java:799)
	at java.lang.ProcessBuilder.start(ProcessBuilder.java:1018)
	at org.apache.hadoop.util.Shell.runCommand(Shell.java:938)
	at org.apache.hadoop.util.Shell.run(Shell.java:901)
	at org.apache.hadoop.fs.DF.getFilesystem(DF.java:74)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker$CheckedVolume.<init>(NameNodeResourceChecker.java:70)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker.addDirToCheck(NameNodeResourceChecker.java:166)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker.<init>(NameNodeResourceChecker.java:135)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startCommonServices(FSNamesystem.java:1168)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.startCommonServices(NameNode.java:791)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.initialize(NameNode.java:717)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.<init>(NameNode.java:940)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.<init>(NameNode.java:913)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.createNameNode(NameNode.java:1646)
	at org.apache.hadoop.hdfs.MiniDFSCluster.createNameNode(MiniDFSCluster.java:1314)
	at org.apache.hadoop.hdfs.MiniDFSCluster.configureNameService(MiniDFSCluster.java:1083)
	at org.apache.hadoop.hdfs.MiniDFSCluster.createNameNodesAndSetConf(MiniDFSCluster.java:958)
	at org.apache.hadoop.hdfs.MiniDFSCluster.initMiniDFSCluster(MiniDFSCluster.java:890)
	at org.apache.hadoop.hdfs.MiniDFSCluster.<init>(MiniDFSCluster.java:518)
	at org.apache.hadoop.hdfs.MiniDFSCluster$Builder.build(MiniDFSCluster.java:477)
	at org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:134)
	at org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:66)
	at org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest.setupClass(HdfsRecoveryZkTest.java:39)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:878)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest

Error Message:
15 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest:     1) Thread[id=9210, name=qtp352575016-9210, state=RUNNABLE, group=TGRP-HdfsRecoveryZkTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$61/398204457.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=9226, name=Socket Reader #1 for port 46252, state=RUNNABLE, group=TGRP-HdfsRecoveryZkTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097)         at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076)    3) Thread[id=9227, name=IPC Server idle connection scanner for port 46252, state=WAITING, group=TGRP-HdfsRecoveryZkTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at java.util.TimerThread.mainLoop(Timer.java:526)         at java.util.TimerThread.run(Timer.java:505)    4) Thread[id=9218, name=Session-HouseKeeper-56c03f08, state=TIMED_WAITING, group=TGRP-HdfsRecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=9207, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-HdfsRecoveryZkTest]         at java.lang.Object.wait(Native Method)         at java.util.TimerThread.mainLoop(Timer.java:552)         at java.util.TimerThread.run(Timer.java:505)    6) Thread[id=9216, name=qtp352575016-9216, state=TIMED_WAITING, group=TGRP-HdfsRecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    7) Thread[id=9209, name=pool-7-thread-1, state=TIMED_WAITING, group=TGRP-HdfsRecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    8) Thread[id=9211, name=qtp352575016-9211, state=RUNNABLE, group=TGRP-HdfsRecoveryZkTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$61/398204457.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)         at java.lang.Thread.run(Thread.java:748)    9) Thread[id=9213, name=qtp352575016-9213, state=TIMED_WAITING, group=TGRP-HdfsRecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)   10) Thread[id=9217, name=qtp352575016-9217, state=TIMED_WAITING, group=TGRP-HdfsRecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)   11) Thread[id=9224, name=FSEditLogAsync, state=WAITING, group=TGRP-HdfsRecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)         at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.dequeueEdit(FSEditLogAsync.java:221)         at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.run(FSEditLogAsync.java:229)         at java.lang.Thread.run(Thread.java:748)   12) Thread[id=9215, name=qtp352575016-9215, state=TIMED_WAITING, group=TGRP-HdfsRecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)   13) Thread[id=9208, name=org.apache.hadoop.util.JvmPauseMonitor$Monitor@1e562220, state=TIMED_WAITING, group=TGRP-HdfsRecoveryZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192)         at java.lang.Thread.run(Thread.java:748)   14) Thread[id=9214, name=qtp352575016-9214, state=TIMED_WAITING, group=TGRP-HdfsRecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)   15) Thread[id=9212, name=qtp352575016-9212-acceptor-0@55a94582-ServerConnector@1cf2f269{HTTP/1.1,[http/1.1]}{lucene2-us-west.apache.org:32906}, state=RUNNABLE, group=TGRP-HdfsRecoveryZkTest]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 15 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest: 
   1) Thread[id=9210, name=qtp352575016-9210, state=RUNNABLE, group=TGRP-HdfsRecoveryZkTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$61/398204457.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=9226, name=Socket Reader #1 for port 46252, state=RUNNABLE, group=TGRP-HdfsRecoveryZkTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097)
        at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076)
   3) Thread[id=9227, name=IPC Server idle connection scanner for port 46252, state=WAITING, group=TGRP-HdfsRecoveryZkTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at java.util.TimerThread.mainLoop(Timer.java:526)
        at java.util.TimerThread.run(Timer.java:505)
   4) Thread[id=9218, name=Session-HouseKeeper-56c03f08, state=TIMED_WAITING, group=TGRP-HdfsRecoveryZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=9207, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-HdfsRecoveryZkTest]
        at java.lang.Object.wait(Native Method)
        at java.util.TimerThread.mainLoop(Timer.java:552)
        at java.util.TimerThread.run(Timer.java:505)
   6) Thread[id=9216, name=qtp352575016-9216, state=TIMED_WAITING, group=TGRP-HdfsRecoveryZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   7) Thread[id=9209, name=pool-7-thread-1, state=TIMED_WAITING, group=TGRP-HdfsRecoveryZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   8) Thread[id=9211, name=qtp352575016-9211, state=RUNNABLE, group=TGRP-HdfsRecoveryZkTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$61/398204457.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
        at java.lang.Thread.run(Thread.java:748)
   9) Thread[id=9213, name=qtp352575016-9213, state=TIMED_WAITING, group=TGRP-HdfsRecoveryZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
  10) Thread[id=9217, name=qtp352575016-9217, state=TIMED_WAITING, group=TGRP-HdfsRecoveryZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
  11) Thread[id=9224, name=FSEditLogAsync, state=WAITING, group=TGRP-HdfsRecoveryZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)
        at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.dequeueEdit(FSEditLogAsync.java:221)
        at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.run(FSEditLogAsync.java:229)
        at java.lang.Thread.run(Thread.java:748)
  12) Thread[id=9215, name=qtp352575016-9215, state=TIMED_WAITING, group=TGRP-HdfsRecoveryZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
  13) Thread[id=9208, name=org.apache.hadoop.util.JvmPauseMonitor$Monitor@1e562220, state=TIMED_WAITING, group=TGRP-HdfsRecoveryZkTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192)
        at java.lang.Thread.run(Thread.java:748)
  14) Thread[id=9214, name=qtp352575016-9214, state=TIMED_WAITING, group=TGRP-HdfsRecoveryZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
  15) Thread[id=9212, name=qtp352575016-9212-acceptor-0@55a94582-ServerConnector@1cf2f269{HTTP/1.1,[http/1.1]}{lucene2-us-west.apache.org:32906}, state=RUNNABLE, group=TGRP-HdfsRecoveryZkTest]
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
        at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385)
        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([DDEF8FCDB3B9404B]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=9210, name=qtp352575016-9210, state=RUNNABLE, group=TGRP-HdfsRecoveryZkTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$61/398204457.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=9226, name=Socket Reader #1 for port 46252, state=RUNNABLE, group=TGRP-HdfsRecoveryZkTest]         at sun.nio.ch.EPollArrayWrapper.interrupt(Native Method)         at sun.nio.ch.EPollArrayWrapper.interrupt(EPollArrayWrapper.java:317)         at sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:207)         at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)         at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097)         at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076)    3) Thread[id=9227, name=IPC Server idle connection scanner for port 46252, state=WAITING, group=TGRP-HdfsRecoveryZkTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at java.util.TimerThread.mainLoop(Timer.java:526)         at java.util.TimerThread.run(Timer.java:505)    4) Thread[id=9218, name=Session-HouseKeeper-56c03f08, state=TIMED_WAITING, group=TGRP-HdfsRecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=9207, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-HdfsRecoveryZkTest]         at java.lang.Object.wait(Native Method)         at java.util.TimerThread.mainLoop(Timer.java:552)         at java.util.TimerThread.run(Timer.java:505)    6) Thread[id=9216, name=qtp352575016-9216, state=TIMED_WAITING, group=TGRP-HdfsRecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    7) Thread[id=9209, name=pool-7-thread-1, state=TIMED_WAITING, group=TGRP-HdfsRecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    8) Thread[id=9211, name=qtp352575016-9211, state=RUNNABLE, group=TGRP-HdfsRecoveryZkTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$61/398204457.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)         at java.lang.Thread.run(Thread.java:748)    9) Thread[id=9213, name=qtp352575016-9213, state=TIMED_WAITING, group=TGRP-HdfsRecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)   10) Thread[id=9217, name=qtp352575016-9217, state=TIMED_WAITING, group=TGRP-HdfsRecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)   11) Thread[id=9215, name=qtp352575016-9215, state=TIMED_WAITING, group=TGRP-HdfsRecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)   12) Thread[id=9214, name=qtp352575016-9214, state=TIMED_WAITING, group=TGRP-HdfsRecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)   13) Thread[id=9212, name=qtp352575016-9212, state=TIMED_WAITING, group=TGRP-HdfsRecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=9210, name=qtp352575016-9210, state=RUNNABLE, group=TGRP-HdfsRecoveryZkTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$61/398204457.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=9226, name=Socket Reader #1 for port 46252, state=RUNNABLE, group=TGRP-HdfsRecoveryZkTest]
        at sun.nio.ch.EPollArrayWrapper.interrupt(Native Method)
        at sun.nio.ch.EPollArrayWrapper.interrupt(EPollArrayWrapper.java:317)
        at sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:207)
        at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
        at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097)
        at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076)
   3) Thread[id=9227, name=IPC Server idle connection scanner for port 

[...truncated too long message...]

.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097)
   [junit4]    >         at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076)
   [junit4]    >    3) Thread[id=9227, name=IPC Server idle connection scanner for port 46252, state=WAITING, group=TGRP-HdfsRecoveryZkTest]
   [junit4]    >         at java.lang.Object.wait(Native Method)
   [junit4]    >         at java.lang.Object.wait(Object.java:502)
   [junit4]    >         at java.util.TimerThread.mainLoop(Timer.java:526)
   [junit4]    >         at java.util.TimerThread.run(Timer.java:505)
   [junit4]    >    4) Thread[id=9218, name=Session-HouseKeeper-56c03f08, state=TIMED_WAITING, group=TGRP-HdfsRecoveryZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    5) Thread[id=9207, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-HdfsRecoveryZkTest]
   [junit4]    >         at java.lang.Object.wait(Native Method)
   [junit4]    >         at java.util.TimerThread.mainLoop(Timer.java:552)
   [junit4]    >         at java.util.TimerThread.run(Timer.java:505)
   [junit4]    >    6) Thread[id=9216, name=qtp352575016-9216, state=TIMED_WAITING, group=TGRP-HdfsRecoveryZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    7) Thread[id=9209, name=pool-7-thread-1, state=TIMED_WAITING, group=TGRP-HdfsRecoveryZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    8) Thread[id=9211, name=qtp352575016-9211, state=RUNNABLE, group=TGRP-HdfsRecoveryZkTest]
   [junit4]    >         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
   [junit4]    >         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
   [junit4]    >         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
   [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$$Lambda$61/398204457.run(Unknown Source)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    9) Thread[id=9213, name=qtp352575016-9213, state=TIMED_WAITING, group=TGRP-HdfsRecoveryZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   10) Thread[id=9217, name=qtp352575016-9217, state=TIMED_WAITING, group=TGRP-HdfsRecoveryZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   11) Thread[id=9215, name=qtp352575016-9215, state=TIMED_WAITING, group=TGRP-HdfsRecoveryZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   12) Thread[id=9214, name=qtp352575016-9214, state=TIMED_WAITING, group=TGRP-HdfsRecoveryZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   13) Thread[id=9212, name=qtp352575016-9212, state=TIMED_WAITING, group=TGRP-HdfsRecoveryZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([DDEF8FCDB3B9404B]:0)
   [junit4] Completed [786/887 (3!)] on J0 in 16.78s, 0 tests, 3 errors <<< FAILURES!

[...truncated 53225 lines...]

[JENKINS] Lucene-Solr-NightlyTests-8.x - Build # 289 - Still Unstable

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-8.x/289/

11 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation

Error Message:
java.security.AccessControlException: access denied ("java.io.FilePermission" "<<ALL FILES>>" "execute")

Stack Trace:
com.google.common.util.concurrent.UncheckedExecutionException: java.security.AccessControlException: access denied ("java.io.FilePermission" "<<ALL FILES>>" "execute")
	at __randomizedtesting.SeedInfo.seed([F872FAB5148988C0]:0)
	at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2050)
	at com.google.common.cache.LocalCache.get(LocalCache.java:3951)
	at com.google.common.cache.LocalCache.getOrLoad(LocalCache.java:3973)
	at com.google.common.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4957)
	at org.apache.hadoop.security.Groups.getGroups(Groups.java:228)
	at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation.getUsersFirstGroup(TestSolrCloudWithSecureImpersonation.java:64)
	at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation.getImpersonatorSettings(TestSolrCloudWithSecureImpersonation.java:87)
	at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation.startup(TestSolrCloudWithSecureImpersonation.java:102)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:878)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.security.AccessControlException: access denied ("java.io.FilePermission" "<<ALL FILES>>" "execute")
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.security.AccessController.checkPermission(AccessController.java:884)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkExec(SecurityManager.java:799)
	at java.lang.ProcessBuilder.start(ProcessBuilder.java:1018)
	at org.apache.hadoop.util.Shell.runCommand(Shell.java:938)
	at org.apache.hadoop.util.Shell.run(Shell.java:901)
	at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1213)
	at org.apache.hadoop.security.ShellBasedUnixGroupsMapping.getUnixGroups(ShellBasedUnixGroupsMapping.java:200)
	at org.apache.hadoop.security.ShellBasedUnixGroupsMapping.getGroups(ShellBasedUnixGroupsMapping.java:97)
	at org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback.getGroups(JniBasedUnixGroupsMappingWithFallback.java:51)
	at org.apache.hadoop.security.Groups$GroupCacheLoader.fetchGroupList(Groups.java:387)
	at org.apache.hadoop.security.Groups$GroupCacheLoader.load(Groups.java:321)
	at org.apache.hadoop.security.Groups$GroupCacheLoader.load(Groups.java:270)
	at com.google.common.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3527)
	at com.google.common.cache.LocalCache$Segment.loadSync(LocalCache.java:2276)
	at com.google.common.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2154)
	at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2044)
	... 31 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation

Error Message:
java.security.AccessControlException: access denied ("java.io.FilePermission" "<<ALL FILES>>" "execute")

Stack Trace:
com.google.common.util.concurrent.UncheckedExecutionException: java.security.AccessControlException: access denied ("java.io.FilePermission" "<<ALL FILES>>" "execute")
	at __randomizedtesting.SeedInfo.seed([F872FAB5148988C0]:0)
	at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2050)
	at com.google.common.cache.LocalCache.get(LocalCache.java:3951)
	at com.google.common.cache.LocalCache.getOrLoad(LocalCache.java:3973)
	at com.google.common.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4957)
	at org.apache.hadoop.security.Groups.getGroups(Groups.java:228)
	at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation.getUsersFirstGroup(TestSolrCloudWithSecureImpersonation.java:64)
	at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation.getImpersonatorSettings(TestSolrCloudWithSecureImpersonation.java:87)
	at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation.shutdown(TestSolrCloudWithSecureImpersonation.java:162)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:901)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.security.AccessControlException: access denied ("java.io.FilePermission" "<<ALL FILES>>" "execute")
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.security.AccessController.checkPermission(AccessController.java:884)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkExec(SecurityManager.java:799)
	at java.lang.ProcessBuilder.start(ProcessBuilder.java:1018)
	at org.apache.hadoop.util.Shell.runCommand(Shell.java:938)
	at org.apache.hadoop.util.Shell.run(Shell.java:901)
	at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1213)
	at org.apache.hadoop.security.ShellBasedUnixGroupsMapping.getUnixGroups(ShellBasedUnixGroupsMapping.java:200)
	at org.apache.hadoop.security.ShellBasedUnixGroupsMapping.getGroups(ShellBasedUnixGroupsMapping.java:97)
	at org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback.getGroups(JniBasedUnixGroupsMappingWithFallback.java:51)
	at org.apache.hadoop.security.Groups$GroupCacheLoader.fetchGroupList(Groups.java:387)
	at org.apache.hadoop.security.Groups$GroupCacheLoader.load(Groups.java:321)
	at org.apache.hadoop.security.Groups$GroupCacheLoader.load(Groups.java:270)
	at com.google.common.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3527)
	at com.google.common.cache.LocalCache$Segment.loadSync(LocalCache.java:2276)
	at com.google.common.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2154)
	at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2044)
	... 30 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.api.collections.TestHdfsCloudBackupRestore

Error Message:
access denied ("java.io.FilePermission" "<<ALL FILES>>" "execute")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "<<ALL FILES>>" "execute")
	at __randomizedtesting.SeedInfo.seed([F872FAB5148988C0]:0)
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.security.AccessController.checkPermission(AccessController.java:884)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkExec(SecurityManager.java:799)
	at java.lang.ProcessBuilder.start(ProcessBuilder.java:1018)
	at org.apache.hadoop.util.Shell.runCommand(Shell.java:938)
	at org.apache.hadoop.util.Shell.run(Shell.java:901)
	at org.apache.hadoop.fs.DF.getFilesystem(DF.java:74)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker$CheckedVolume.<init>(NameNodeResourceChecker.java:70)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker.addDirToCheck(NameNodeResourceChecker.java:166)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker.<init>(NameNodeResourceChecker.java:135)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startCommonServices(FSNamesystem.java:1168)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.startCommonServices(NameNode.java:791)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.initialize(NameNode.java:717)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.<init>(NameNode.java:940)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.<init>(NameNode.java:913)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.createNameNode(NameNode.java:1646)
	at org.apache.hadoop.hdfs.MiniDFSCluster.createNameNode(MiniDFSCluster.java:1314)
	at org.apache.hadoop.hdfs.MiniDFSCluster.configureNameService(MiniDFSCluster.java:1083)
	at org.apache.hadoop.hdfs.MiniDFSCluster.createNameNodesAndSetConf(MiniDFSCluster.java:958)
	at org.apache.hadoop.hdfs.MiniDFSCluster.initMiniDFSCluster(MiniDFSCluster.java:890)
	at org.apache.hadoop.hdfs.MiniDFSCluster.<init>(MiniDFSCluster.java:518)
	at org.apache.hadoop.hdfs.MiniDFSCluster$Builder.build(MiniDFSCluster.java:477)
	at org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:134)
	at org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:66)
	at org.apache.solr.cloud.api.collections.TestHdfsCloudBackupRestore.setupClass(TestHdfsCloudBackupRestore.java:110)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:878)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.api.collections.TestHdfsCloudBackupRestore

Error Message:
15 threads leaked from SUITE scope at org.apache.solr.cloud.api.collections.TestHdfsCloudBackupRestore:     1) Thread[id=9439, name=qtp757751663-9439, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=9436, name=qtp757751663-9436, state=RUNNABLE, group=TGRP-TestHdfsCloudBackupRestore]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$61/1720399233.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=9437, name=qtp757751663-9437, state=RUNNABLE, group=TGRP-TestHdfsCloudBackupRestore]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$61/1720399233.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=9442, name=qtp757751663-9442, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=9453, name=IPC Server idle connection scanner for port 36999, state=WAITING, group=TGRP-TestHdfsCloudBackupRestore]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at java.util.TimerThread.mainLoop(Timer.java:526)         at java.util.TimerThread.run(Timer.java:505)    6) Thread[id=9440, name=qtp757751663-9440, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    7) Thread[id=9452, name=Socket Reader #1 for port 36999, state=RUNNABLE, group=TGRP-TestHdfsCloudBackupRestore]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097)         at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076)    8) Thread[id=9443, name=qtp757751663-9443, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    9) Thread[id=9433, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]         at java.lang.Object.wait(Native Method)         at java.util.TimerThread.mainLoop(Timer.java:552)         at java.util.TimerThread.run(Timer.java:505)   10) Thread[id=9434, name=org.apache.hadoop.util.JvmPauseMonitor$Monitor@27fe5174, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]         at java.lang.Thread.sleep(Native Method)         at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192)         at java.lang.Thread.run(Thread.java:748)   11) Thread[id=9450, name=FSEditLogAsync, state=WAITING, group=TGRP-TestHdfsCloudBackupRestore]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)         at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.dequeueEdit(FSEditLogAsync.java:221)         at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.run(FSEditLogAsync.java:229)         at java.lang.Thread.run(Thread.java:748)   12) Thread[id=9438, name=qtp757751663-9438-acceptor-0@757aa44f-ServerConnector@58778cf6{HTTP/1.1,[http/1.1]}{localhost.localdomain:34359}, state=RUNNABLE, group=TGRP-TestHdfsCloudBackupRestore]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)         at java.lang.Thread.run(Thread.java:748)   13) Thread[id=9444, name=Session-HouseKeeper-776711cb, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   14) Thread[id=9435, name=pool-9-thread-1, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   15) Thread[id=9441, name=qtp757751663-9441, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 15 threads leaked from SUITE scope at org.apache.solr.cloud.api.collections.TestHdfsCloudBackupRestore: 
   1) Thread[id=9439, name=qtp757751663-9439, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=9436, name=qtp757751663-9436, state=RUNNABLE, group=TGRP-TestHdfsCloudBackupRestore]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$61/1720399233.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=9437, name=qtp757751663-9437, state=RUNNABLE, group=TGRP-TestHdfsCloudBackupRestore]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$61/1720399233.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=9442, name=qtp757751663-9442, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=9453, name=IPC Server idle connection scanner for port 36999, state=WAITING, group=TGRP-TestHdfsCloudBackupRestore]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at java.util.TimerThread.mainLoop(Timer.java:526)
        at java.util.TimerThread.run(Timer.java:505)
   6) Thread[id=9440, name=qtp757751663-9440, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   7) Thread[id=9452, name=Socket Reader #1 for port 36999, state=RUNNABLE, group=TGRP-TestHdfsCloudBackupRestore]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097)
        at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076)
   8) Thread[id=9443, name=qtp757751663-9443, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   9) Thread[id=9433, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]
        at java.lang.Object.wait(Native Method)
        at java.util.TimerThread.mainLoop(Timer.java:552)
        at java.util.TimerThread.run(Timer.java:505)
  10) Thread[id=9434, name=org.apache.hadoop.util.JvmPauseMonitor$Monitor@27fe5174, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192)
        at java.lang.Thread.run(Thread.java:748)
  11) Thread[id=9450, name=FSEditLogAsync, state=WAITING, group=TGRP-TestHdfsCloudBackupRestore]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)
        at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.dequeueEdit(FSEditLogAsync.java:221)
        at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.run(FSEditLogAsync.java:229)
        at java.lang.Thread.run(Thread.java:748)
  12) Thread[id=9438, name=qtp757751663-9438-acceptor-0@757aa44f-ServerConnector@58778cf6{HTTP/1.1,[http/1.1]}{localhost.localdomain:34359}, state=RUNNABLE, group=TGRP-TestHdfsCloudBackupRestore]
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
        at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385)
        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
        at java.lang.Thread.run(Thread.java:748)
  13) Thread[id=9444, name=Session-HouseKeeper-776711cb, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  14) Thread[id=9435, name=pool-9-thread-1, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  15) Thread[id=9441, name=qtp757751663-9441, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([F872FAB5148988C0]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.api.collections.TestHdfsCloudBackupRestore

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=9439, name=qtp757751663-9439, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=9436, name=qtp757751663-9436, state=RUNNABLE, group=TGRP-TestHdfsCloudBackupRestore]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$61/1720399233.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=9437, name=qtp757751663-9437, state=RUNNABLE, group=TGRP-TestHdfsCloudBackupRestore]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$61/1720399233.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=9442, name=qtp757751663-9442, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=9453, name=IPC Server idle connection scanner for port 36999, state=WAITING, group=TGRP-TestHdfsCloudBackupRestore]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at java.util.TimerThread.mainLoop(Timer.java:526)         at java.util.TimerThread.run(Timer.java:505)    6) Thread[id=9440, name=qtp757751663-9440, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    7) Thread[id=9452, name=Socket Reader #1 for port 36999, state=RUNNABLE, group=TGRP-TestHdfsCloudBackupRestore]         at sun.nio.ch.EPollArrayWrapper.interrupt(Native Method)         at sun.nio.ch.EPollArrayWrapper.interrupt(EPollArrayWrapper.java:317)         at sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:207)         at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)         at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097)         at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076)    8) Thread[id=9443, name=qtp757751663-9443, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    9) Thread[id=9433, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]         at java.lang.Object.wait(Native Method)         at java.util.TimerThread.mainLoop(Timer.java:552)         at java.util.TimerThread.run(Timer.java:505)   10) Thread[id=9438, name=qtp757751663-9438, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)   11) Thread[id=9444, name=Session-HouseKeeper-776711cb, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   12) Thread[id=9435, name=pool-9-thread-1, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   13) Thread[id=9441, name=qtp757751663-9441, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=9439, name=qtp757751663-9439, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=9436, name=qtp757751663-9436, state=RUNNABLE, group=TGRP-TestHdfsCloudBackupRestore]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$61/1720399233.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=9437, name=qtp757751663-9437, state=RUNNABLE, group=TGRP-TestHdfsCloudBackupRestore]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$61/1720399233.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=9442, name=qtp757751663-9442, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=9453, name=IPC Server idle connection scanner for port 36999, state=WAITING, group=TGRP-TestHdfsCloudBackupRestore]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at java.util.TimerThread.mainLoop(Timer.java:526)
        at java.util.TimerThread.run(Timer.java:505)
   6) Thread[id=9440, name=qtp757751663-9440, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   7) Thread[id=9452, name=Socket Reader #1 for port 36999, state=RUNNABLE, group=TGRP-TestHdfsCloudBackupRestore]
        at sun.nio.ch.EPollArrayWrapper.interrupt(Native Method)
        at sun.nio.ch.EPollArrayWrapper.interrupt(EPollArrayWrapper.java:317)
        at sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:207)
        at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
        at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097)
        at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076)
   8) Thread[id=9443, name=qtp757751663-9443, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   9) Thread[id=9433, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]
        at java.lang.Object.wait(Native Method)
        at java.util.TimerThread.mainLoop(Timer.java:552)
        at java.util.TimerThread.run(Timer.java:505)
  10) Thread[id=9438, name=qtp757751663-9438, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
  11) Thread[id=9444, name=Session-HouseKeeper-776711cb, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  12) Thread[id=9435, name=pool-9-thread-1, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  13) Thread[id=9441, name=qtp757751663-9441, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([F872FAB5148988C0]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsBasicDistributedZkTest

Error Message:
access denied ("java.io.FilePermission" "<<ALL FILES>>" "execute")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "<<ALL FILES>>" "execute")
	at __randomizedtesting.SeedInfo.seed([F872FAB5148988C0]:0)
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.security.AccessController.checkPermission(AccessController.java:884)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkExec(SecurityManager.java:799)
	at java.lang.ProcessBuilder.start(ProcessBuilder.java:1018)
	at org.apache.hadoop.util.Shell.runCommand(Shell.java:938)
	at org.apache.hadoop.util.Shell.run(Shell.java:901)
	at org.apache.hadoop.fs.DF.getFilesystem(DF.java:74)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker$CheckedVolume.<init>(NameNodeResourceChecker.java:70)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker.addDirToCheck(NameNodeResourceChecker.java:166)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker.<init>(NameNodeResourceChecker.java:135)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startCommonServices(FSNamesystem.java:1168)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.startCommonServices(NameNode.java:791)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.initialize(NameNode.java:717)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.<init>(NameNode.java:940)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.<init>(NameNode.java:913)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.createNameNode(NameNode.java:1646)
	at org.apache.hadoop.hdfs.MiniDFSCluster.createNameNode(MiniDFSCluster.java:1314)
	at org.apache.hadoop.hdfs.MiniDFSCluster.configureNameService(MiniDFSCluster.java:1083)
	at org.apache.hadoop.hdfs.MiniDFSCluster.createNameNodesAndSetConf(MiniDFSCluster.java:958)
	at org.apache.hadoop.hdfs.MiniDFSCluster.initMiniDFSCluster(MiniDFSCluster.java:890)
	at org.apache.hadoop.hdfs.MiniDFSCluster.<init>(MiniDFSCluster.java:518)
	at org.apache.hadoop.hdfs.MiniDFSCluster$Builder.build(MiniDFSCluster.java:477)
	at org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:134)
	at org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:66)
	at org.apache.solr.cloud.hdfs.HdfsBasicDistributedZkTest.setupClass(HdfsBasicDistributedZkTest.java:41)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:878)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsBasicDistributedZkTest

Error Message:
15 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsBasicDistributedZkTest:     1) Thread[id=60364, name=pool-7-thread-1, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=60381, name=Socket Reader #1 for port 44983, state=RUNNABLE, group=TGRP-HdfsBasicDistributedZkTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097)         at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076)    3) Thread[id=60369, name=qtp973505396-60369, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=60371, name=qtp973505396-60371, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=60368, name=qtp973505396-60368, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=60366, name=qtp973505396-60366, state=RUNNABLE, group=TGRP-HdfsBasicDistributedZkTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$61/654266953.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)         at java.lang.Thread.run(Thread.java:748)    7) Thread[id=60372, name=qtp973505396-60372, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)    8) Thread[id=60367, name=qtp973505396-60367-acceptor-0@2ec50659-ServerConnector@868fade{HTTP/1.1,[http/1.1]}{localhost.localdomain:35615}, state=RUNNABLE, group=TGRP-HdfsBasicDistributedZkTest]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)         at java.lang.Thread.run(Thread.java:748)    9) Thread[id=60370, name=qtp973505396-60370, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)         at java.lang.Thread.run(Thread.java:748)   10) Thread[id=60373, name=Session-HouseKeeper-7bee775d, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   11) Thread[id=60365, name=qtp973505396-60365, state=RUNNABLE, group=TGRP-HdfsBasicDistributedZkTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$61/654266953.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)         at java.lang.Thread.run(Thread.java:748)   12) Thread[id=60362, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]         at java.lang.Object.wait(Native Method)         at java.util.TimerThread.mainLoop(Timer.java:552)         at java.util.TimerThread.run(Timer.java:505)   13) Thread[id=60379, name=FSEditLogAsync, state=WAITING, group=TGRP-HdfsBasicDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)         at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.dequeueEdit(FSEditLogAsync.java:221)         at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.run(FSEditLogAsync.java:229)         at java.lang.Thread.run(Thread.java:748)   14) Thread[id=60363, name=org.apache.hadoop.util.JvmPauseMonitor$Monitor@37f57bec, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192)         at java.lang.Thread.run(Thread.java:748)   15) Thread[id=60382, name=IPC Server idle connection scanner for port 44983, state=WAITING, group=TGRP-HdfsBasicDistributedZkTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at java.util.TimerThread.mainLoop(Timer.java:526)         at java.util.TimerThread.run(Timer.java:505)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 15 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsBasicDistributedZkTest: 
   1) Thread[id=60364, name=pool-7-thread-1, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=60381, name=Socket Reader #1 for port 44983, state=RUNNABLE, group=TGRP-HdfsBasicDistributedZkTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097)
        at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076)
   3) Thread[id=60369, name=qtp973505396-60369, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=60371, name=qtp973505396-60371, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=60368, name=qtp973505396-60368, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=60366, name=qtp973505396-60366, state=RUNNABLE, group=TGRP-HdfsBasicDistributedZkTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$61/654266953.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
        at java.lang.Thread.run(Thread.java:748)
   7) Thread[id=60372, name=qtp973505396-60372, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
   8) Thread[id=60367, name=qtp973505396-60367-acceptor-0@2ec50659-ServerConnector@868fade{HTTP/1.1,[http/1.1]}{localhost.localdomain:35615}, state=RUNNABLE, group=TGRP-HdfsBasicDistributedZkTest]
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
        at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385)
        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
        at java.lang.Thread.run(Thread.java:748)
   9) Thread[id=60370, name=qtp973505396-60370, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
        at java.lang.Thread.run(Thread.java:748)
  10) Thread[id=60373, name=Session-HouseKeeper-7bee775d, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  11) Thread[id=60365, name=qtp973505396-60365, state=RUNNABLE, group=TGRP-HdfsBasicDistributedZkTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$61/654266953.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
        at java.lang.Thread.run(Thread.java:748)
  12) Thread[id=60362, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]
        at java.lang.Object.wait(Native Method)
        at java.util.TimerThread.mainLoop(Timer.java:552)
        at java.util.TimerThread.run(Timer.java:505)
  13) Thread[id=60379, name=FSEditLogAsync, state=WAITING, group=TGRP-HdfsBasicDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)
        at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.dequeueEdit(FSEditLogAsync.java:221)
        at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.run(FSEditLogAsync.java:229)
        at java.lang.Thread.run(Thread.java:748)
  14) Thread[id=60363, name=org.apache.hadoop.util.JvmPauseMonitor$Monitor@37f57bec, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192)
        at java.lang.Thread.run(Thread.java:748)
  15) Thread[id=60382, name=IPC Server idle connection scanner for port 44983, state=WAITING, group=TGRP-HdfsBasicDistributedZkTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at java.util.TimerThread.mainLoop(Timer.java:526)
        at java.util.TimerThread.run(Timer.java:505)
	at __randomizedtesting.SeedInfo.seed([F872FAB5148988C0]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsBasicDistributedZkTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=60364, name=pool-7-thread-1, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=60381, name=Socket Reader #1 for port 44983, state=RUNNABLE, group=TGRP-HdfsBasicDistributedZkTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097)         at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076)    3) Thread[id=60369, name=qtp973505396-60369, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         

[...truncated too long message...]

.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    4) Thread[id=60371, name=qtp973505396-60371, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    5) Thread[id=60368, name=qtp973505396-60368, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    6) Thread[id=60366, name=qtp973505396-60366, state=RUNNABLE, group=TGRP-HdfsBasicDistributedZkTest]
   [junit4]    >         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
   [junit4]    >         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
   [junit4]    >         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
   [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$$Lambda$61/654266953.run(Unknown Source)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    7) Thread[id=60372, name=qtp973505396-60372, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    8) Thread[id=60367, name=qtp973505396-60367, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    9) Thread[id=60370, name=qtp973505396-60370, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   10) Thread[id=60373, name=Session-HouseKeeper-7bee775d, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   11) Thread[id=60365, name=qtp973505396-60365, state=RUNNABLE, group=TGRP-HdfsBasicDistributedZkTest]
   [junit4]    >         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
   [junit4]    >         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
   [junit4]    >         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
   [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$$Lambda$61/654266953.run(Unknown Source)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   12) Thread[id=60362, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]
   [junit4]    >         at java.lang.Object.wait(Native Method)
   [junit4]    >         at java.util.TimerThread.mainLoop(Timer.java:552)
   [junit4]    >         at java.util.TimerThread.run(Timer.java:505)
   [junit4]    >   13) Thread[id=60382, name=IPC Server idle connection scanner for port 44983, state=WAITING, group=TGRP-HdfsBasicDistributedZkTest]
   [junit4]    >         at java.lang.Object.wait(Native Method)
   [junit4]    >         at java.lang.Object.wait(Object.java:502)
   [junit4]    >         at java.util.TimerThread.mainLoop(Timer.java:526)
   [junit4]    >         at java.util.TimerThread.run(Timer.java:505)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([F872FAB5148988C0]:0)
   [junit4] Completed [806/887 (4!)] on J1 in 14.68s, 0 tests, 3 errors <<< FAILURES!

[...truncated 53150 lines...]

[JENKINS] Lucene-Solr-NightlyTests-8.x - Build # 288 - Still Unstable

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-8.x/288/

1 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.autoscaling.SearchRateTriggerIntegrationTest

Error Message:
Clean up static fields (in @AfterClass?) and null them, your test still has references to classes of which the sizes cannot be measured due to security restrictions or Java 9 module encapsulation:   - private static org.apache.solr.client.solrj.cloud.SolrCloudManager org.apache.solr.cloud.autoscaling.SearchRateTriggerIntegrationTest.cloudManager

Stack Trace:
junit.framework.AssertionFailedError: Clean up static fields (in @AfterClass?) and null them, your test still has references to classes of which the sizes cannot be measured due to security restrictions or Java 9 module encapsulation:
  - private static org.apache.solr.client.solrj.cloud.SolrCloudManager org.apache.solr.cloud.autoscaling.SearchRateTriggerIntegrationTest.cloudManager
	at __randomizedtesting.SeedInfo.seed([23CA321706FE3269]:0)
	at com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule$1.afterAlways(StaticFieldsInvariantRule.java:146)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.security.AccessControlException: access denied ("java.lang.RuntimePermission" "accessClassInPackage.sun.nio.fs")
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.security.AccessController.checkPermission(AccessController.java:884)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkPackageAccess(SecurityManager.java:1564)
	at java.lang.Class.checkPackageAccess(Class.java:2372)
	at java.lang.Class.checkMemberAccess(Class.java:2351)
	at java.lang.Class.getDeclaredFields(Class.java:1915)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator$2.run(RamUsageEstimator.java:585)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator$2.run(RamUsageEstimator.java:582)
	at java.security.AccessController.doPrivileged(Native Method)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.createCacheEntry(RamUsageEstimator.java:582)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.measureSizeOf(RamUsageEstimator.java:545)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.sizeOfAll(RamUsageEstimator.java:387)
	at com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule$1.afterAlways(StaticFieldsInvariantRule.java:129)
	... 10 more




Build Log:
[...truncated 15141 lines...]
   [junit4] Suite: org.apache.solr.cloud.autoscaling.SearchRateTriggerIntegrationTest
   [junit4]   2> 3574249 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[23CA321706FE3269]-worker) [     ] o.a.s.SolrTestCaseJ4 Created dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.autoscaling.SearchRateTriggerIntegrationTest_23CA321706FE3269-001/data-dir-187-001
   [junit4]   2> 3574249 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[23CA321706FE3269]-worker) [     ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 3574250 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[23CA321706FE3269]-worker) [     ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 3574250 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[23CA321706FE3269]-worker) [     ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 3574250 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[23CA321706FE3269]-worker) [     ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 5 servers in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.autoscaling.SearchRateTriggerIntegrationTest_23CA321706FE3269-001/tempDir-001
   [junit4]   2> 3574251 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[23CA321706FE3269]-worker) [     ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 3574251 INFO  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 3574251 INFO  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 3574351 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[23CA321706FE3269]-worker) [     ] o.a.s.c.ZkTestServer start zk server on port:38317
   [junit4]   2> 3574351 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[23CA321706FE3269]-worker) [     ] o.a.s.c.ZkTestServer waitForServerUp: 127.0.0.1:38317
   [junit4]   2> 3574351 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[23CA321706FE3269]-worker) [     ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:38317
   [junit4]   2> 3574351 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[23CA321706FE3269]-worker) [     ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 38317
   [junit4]   2> 3574352 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[23CA321706FE3269]-worker) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 3574361 INFO  (zkConnectionManagerCallback-17307-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3574361 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[23CA321706FE3269]-worker) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 3574365 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[23CA321706FE3269]-worker) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 3574365 INFO  (zkConnectionManagerCallback-17309-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3574365 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[23CA321706FE3269]-worker) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 3574366 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[23CA321706FE3269]-worker) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 3574366 INFO  (zkConnectionManagerCallback-17311-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3574366 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[23CA321706FE3269]-worker) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 3574468 WARN  (jetty-launcher-17312-thread-1) [     ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 3574468 WARN  (jetty-launcher-17312-thread-5) [     ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 3574468 WARN  (jetty-launcher-17312-thread-3) [     ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 3574468 WARN  (jetty-launcher-17312-thread-4) [     ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 3574469 INFO  (jetty-launcher-17312-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 3574469 INFO  (jetty-launcher-17312-thread-5) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 3574469 INFO  (jetty-launcher-17312-thread-4) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 3574469 INFO  (jetty-launcher-17312-thread-3) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 3574469 INFO  (jetty-launcher-17312-thread-5) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 3574469 INFO  (jetty-launcher-17312-thread-4) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 3574469 INFO  (jetty-launcher-17312-thread-3) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 3574469 INFO  (jetty-launcher-17312-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 3574469 INFO  (jetty-launcher-17312-thread-4) [     ] o.e.j.s.Server jetty-9.4.19.v20190610; built: 2019-06-10T16:30:51.723Z; git: afcf563148970e98786327af5e07c261fda175d3; jvm 1.8.0_191-b12
   [junit4]   2> 3574469 INFO  (jetty-launcher-17312-thread-3) [     ] o.e.j.s.Server jetty-9.4.19.v20190610; built: 2019-06-10T16:30:51.723Z; git: afcf563148970e98786327af5e07c261fda175d3; jvm 1.8.0_191-b12
   [junit4]   2> 3574469 INFO  (jetty-launcher-17312-thread-1) [     ] o.e.j.s.Server jetty-9.4.19.v20190610; built: 2019-06-10T16:30:51.723Z; git: afcf563148970e98786327af5e07c261fda175d3; jvm 1.8.0_191-b12
   [junit4]   2> 3574469 INFO  (jetty-launcher-17312-thread-5) [     ] o.e.j.s.Server jetty-9.4.19.v20190610; built: 2019-06-10T16:30:51.723Z; git: afcf563148970e98786327af5e07c261fda175d3; jvm 1.8.0_191-b12
   [junit4]   2> 3574469 WARN  (jetty-launcher-17312-thread-2) [     ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 3574469 INFO  (jetty-launcher-17312-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 3574469 INFO  (jetty-launcher-17312-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 3574469 INFO  (jetty-launcher-17312-thread-2) [     ] o.e.j.s.Server jetty-9.4.19.v20190610; built: 2019-06-10T16:30:51.723Z; git: afcf563148970e98786327af5e07c261fda175d3; jvm 1.8.0_191-b12
   [junit4]   2> 3574469 INFO  (jetty-launcher-17312-thread-3) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 3574469 INFO  (jetty-launcher-17312-thread-3) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 3574469 INFO  (jetty-launcher-17312-thread-3) [     ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 3574471 INFO  (jetty-launcher-17312-thread-3) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@56b49c24{/solr,null,AVAILABLE}
   [junit4]   2> 3574472 INFO  (jetty-launcher-17312-thread-1) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 3574472 INFO  (jetty-launcher-17312-thread-1) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 3574472 INFO  (jetty-launcher-17312-thread-1) [     ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 3574472 INFO  (jetty-launcher-17312-thread-2) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 3574472 INFO  (jetty-launcher-17312-thread-2) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 3574472 INFO  (jetty-launcher-17312-thread-2) [     ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 3574472 INFO  (jetty-launcher-17312-thread-5) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 3574472 INFO  (jetty-launcher-17312-thread-5) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 3574472 INFO  (jetty-launcher-17312-thread-3) [     ] o.e.j.s.AbstractConnector Started ServerConnector@70d8d564{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:40305}
   [junit4]   2> 3574472 INFO  (jetty-launcher-17312-thread-5) [     ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 3574472 INFO  (jetty-launcher-17312-thread-3) [     ] o.e.j.s.Server Started @3574501ms
   [junit4]   2> 3574472 INFO  (jetty-launcher-17312-thread-3) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=40305}
   [junit4]   2> 3574472 INFO  (jetty-launcher-17312-thread-1) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6985c470{/solr,null,AVAILABLE}
   [junit4]   2> 3574472 ERROR (jetty-launcher-17312-thread-3) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 3574472 INFO  (jetty-launcher-17312-thread-3) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 3574472 INFO  (jetty-launcher-17312-thread-3) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solrâ„¢ version 8.4.0
   [junit4]   2> 3574472 INFO  (jetty-launcher-17312-thread-3) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 3574472 INFO  (jetty-launcher-17312-thread-3) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 3574472 INFO  (jetty-launcher-17312-thread-3) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-12-04T18:34:27.483Z
   [junit4]   2> 3574472 INFO  (jetty-launcher-17312-thread-1) [     ] o.e.j.s.AbstractConnector Started ServerConnector@33ee1f34{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:46051}
   [junit4]   2> 3574472 INFO  (jetty-launcher-17312-thread-1) [     ] o.e.j.s.Server Started @3574502ms
   [junit4]   2> 3574472 INFO  (jetty-launcher-17312-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=46051}
   [junit4]   2> 3574473 ERROR (jetty-launcher-17312-thread-1) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 3574473 INFO  (jetty-launcher-17312-thread-1) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 3574473 INFO  (jetty-launcher-17312-thread-1) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solrâ„¢ version 8.4.0
   [junit4]   2> 3574473 INFO  (jetty-launcher-17312-thread-1) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 3574473 INFO  (jetty-launcher-17312-thread-1) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 3574473 INFO  (jetty-launcher-17312-thread-1) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-12-04T18:34:27.484Z
   [junit4]   2> 3574473 INFO  (jetty-launcher-17312-thread-3) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 3574473 INFO  (jetty-launcher-17312-thread-5) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@682e9147{/solr,null,AVAILABLE}
   [junit4]   2> 3574474 INFO  (jetty-launcher-17312-thread-4) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 3574474 INFO  (jetty-launcher-17312-thread-4) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 3574474 INFO  (jetty-launcher-17312-thread-4) [     ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 3574474 INFO  (jetty-launcher-17312-thread-2) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@36b2112c{/solr,null,AVAILABLE}
   [junit4]   2> 3574474 INFO  (jetty-launcher-17312-thread-1) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 3574474 INFO  (jetty-launcher-17312-thread-5) [     ] o.e.j.s.AbstractConnector Started ServerConnector@1b26966d{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:38281}
   [junit4]   2> 3574474 INFO  (jetty-launcher-17312-thread-2) [     ] o.e.j.s.AbstractConnector Started ServerConnector@5b4c30a2{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:40565}
   [junit4]   2> 3574474 INFO  (jetty-launcher-17312-thread-5) [     ] o.e.j.s.Server Started @3574503ms
   [junit4]   2> 3574474 INFO  (jetty-launcher-17312-thread-2) [     ] o.e.j.s.Server Started @3574503ms
   [junit4]   2> 3574474 INFO  (jetty-launcher-17312-thread-5) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=38281}
   [junit4]   2> 3574474 INFO  (jetty-launcher-17312-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=40565}
   [junit4]   2> 3574474 INFO  (jetty-launcher-17312-thread-4) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4a482a54{/solr,null,AVAILABLE}
   [junit4]   2> 3574474 ERROR (jetty-launcher-17312-thread-2) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 3574474 ERROR (jetty-launcher-17312-thread-5) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 3574474 INFO  (jetty-launcher-17312-thread-2) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 3574474 INFO  (jetty-launcher-17312-thread-5) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 3574474 INFO  (jetty-launcher-17312-thread-2) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solrâ„¢ version 8.4.0
   [junit4]   2> 3574474 INFO  (jetty-launcher-17312-thread-5) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solrâ„¢ version 8.4.0
   [junit4]   2> 3574474 INFO  (jetty-launcher-17312-thread-2) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 3574474 INFO  (jetty-launcher-17312-thread-5) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 3574474 INFO  (jetty-launcher-17312-thread-2) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 3574474 INFO  (jetty-launcher-17312-thread-5) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 3574474 INFO  (jetty-launcher-17312-thread-2) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-12-04T18:34:27.485Z
   [junit4]   2> 3574474 INFO  (jetty-launcher-17312-thread-5) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-12-04T18:34:27.485Z
   [junit4]   2> 3574475 INFO  (zkConnectionManagerCallback-17316-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3574475 INFO  (jetty-launcher-17312-thread-1) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 3574475 INFO  (zkConnectionManagerCallback-17314-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3574475 INFO  (jetty-launcher-17312-thread-3) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 3574476 INFO  (jetty-launcher-17312-thread-3) [     ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 3574476 INFO  (jetty-launcher-17312-thread-5) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 3574476 INFO  (jetty-launcher-17312-thread-1) [     ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 3574476 INFO  (jetty-launcher-17312-thread-4) [     ] o.e.j.s.AbstractConnector Started ServerConnector@4cd19a38{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:46631}
   [junit4]   2> 3574476 INFO  (jetty-launcher-17312-thread-4) [     ] o.e.j.s.Server Started @3574505ms
   [junit4]   2> 3574476 INFO  (jetty-launcher-17312-thread-4) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=46631}
   [junit4]   2> 3574476 ERROR (jetty-launcher-17312-thread-4) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 3574476 INFO  (jetty-launcher-17312-thread-4) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 3574476 INFO  (jetty-launcher-17312-thread-4) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solrâ„¢ version 8.4.0
   [junit4]   2> 3574476 INFO  (jetty-launcher-17312-thread-4) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 3574476 INFO  (jetty-launcher-17312-thread-4) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 3574476 INFO  (jetty-launcher-17312-thread-4) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-12-04T18:34:27.487Z
   [junit4]   2> 3574480 INFO  (jetty-launcher-17312-thread-2) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 3574481 INFO  (jetty-launcher-17312-thread-4) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 3574481 INFO  (zkConnectionManagerCallback-17320-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3574481 INFO  (zkConnectionManagerCallback-17318-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3574481 INFO  (jetty-launcher-17312-thread-2) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 3574481 INFO  (zkConnectionManagerCallback-17322-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3574481 INFO  (jetty-launcher-17312-thread-5) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 3574481 INFO  (jetty-launcher-17312-thread-4) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 3574582 INFO  (jetty-launcher-17312-thread-2) [     ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 3574582 INFO  (jetty-launcher-17312-thread-5) [     ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 3574582 INFO  (jetty-launcher-17312-thread-4) [     ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 3574638 INFO  (jetty-launcher-17312-thread-1) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
   [junit4]   2> 3574638 INFO  (jetty-launcher-17312-thread-3) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
   [junit4]   2> 3574639 WARN  (jetty-launcher-17312-thread-1) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@a5123e5[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 3574639 WARN  (jetty-launcher-17312-thread-3) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@3499539c[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 3574639 WARN  (jetty-launcher-17312-thread-3) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@3499539c[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 3574639 WARN  (jetty-launcher-17312-thread-1) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@a5123e5[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 3574641 WARN  (jetty-launcher-17312-thread-3) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@3d383948[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 3574641 WARN  (jetty-launcher-17312-thread-3) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@3d383948[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 3574641 WARN  (jetty-launcher-17312-thread-1) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@782d9ff2[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 3574641 WARN  (jetty-launcher-17312-thread-1) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@782d9ff2[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 3574641 INFO  (jetty-launcher-17312-thread-1) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38317/solr
   [junit4]   2> 3574641 INFO  (jetty-launcher-17312-thread-3) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38317/solr
   [junit4]   2> 3574642 INFO  (jetty-launcher-17312-thread-1) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 3574643 INFO  (jetty-launcher-17312-thread-3) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 3574643 INFO  (zkConnectionManagerCallback-17335-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3574643 INFO  (jetty-launcher-17312-thread-1) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 3574643 INFO  (zkConnectionManagerCallback-17336-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3574643 INFO  (jetty-launcher-17312-thread-3) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 3574752 INFO  (jetty-launcher-17312-thread-4) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
   [junit4]   2> 3574753 WARN  (jetty-launcher-17312-thread-4) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@3d26e404[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 3574753 WARN  (jetty-launcher-17312-thread-4) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@3d26e404[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 3574755 WARN  (jetty-launcher-17312-thread-4) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@27f29839[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 3574755 WARN  (jetty-launcher-17312-thread-4) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@27f29839[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 3574756 INFO  (jetty-launcher-17312-thread-4) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38317/solr
   [junit4]   2> 3574756 INFO  (jetty-launcher-17312-thread-4) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 3574757 INFO  (zkConnectionManagerCallback-17343-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3574757 INFO  (jetty-launcher-17312-thread-4) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 3574860 INFO  (jetty-launcher-17312-thread-2) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
   [junit4]   2> 3574860 WARN  (jetty-launcher-17312-thread-2) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@e81c34e[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 3574860 WARN  (jetty-launcher-17312-thread-2) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@e81c34e[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 3574862 WARN  (jetty-launcher-17312-thread-2) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@11585dd0[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 3574862 WARN  (jetty-launcher-17312-thread-2) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@11585dd0[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 3574863 INFO  (jetty-launcher-17312-thread-2) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38317/solr
   [junit4]   2> 3574863 INFO  (jetty-launcher-17312-thread-2) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 3574864 INFO  (zkConnectionManagerCallback-17351-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3574864 INFO  (jetty-launcher-17312-thread-2) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 3574894 INFO  (jetty-launcher-17312-thread-1) [n:127.0.0.1:46051_solr     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 3574895 INFO  (zkConnectionManagerCallback-17353-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3574895 INFO  (jetty-launcher-17312-thread-1) [n:127.0.0.1:46051_solr     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 3574943 INFO  (jetty-launcher-17312-thread-5) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
   [junit4]   2> 3574944 WARN  (jetty-launcher-17312-thread-5) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@3e4e7e60[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 3574944 WARN  (jetty-launcher-17312-thread-5) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@3e4e7e60[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 3574945 WARN  (jetty-launcher-17312-thread-5) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@295c0347[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 3574945 WARN  (jetty-launcher-17312-thread-5) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@295c0347[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 3574946 INFO  (jetty-launcher-17312-thread-5) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38317/solr
   [junit4]   2> 3574947 INFO  (jetty-launcher-17312-thread-5) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 3574948 INFO  (zkConnectionManagerCallback-17359-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3574948 INFO  (jetty-launcher-17312-thread-5) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 3574994 INFO  (jetty-launcher-17312-thread-4) [n:127.0.0.1:46631_solr     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 3574994 INFO  (jetty-launcher-17312-thread-3) [n:127.0.0.1:40305_solr     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 3574995 INFO  (zkConnectionManagerCallback-17361-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3574995 INFO  (jetty-launcher-17312-thread-4) [n:127.0.0.1:46631_solr     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 3574995 INFO  (zkConnectionManagerCallback-17363-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3574995 INFO  (jetty-launcher-17312-thread-3) [n:127.0.0.1:40305_solr     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 3575095 INFO  (jetty-launcher-17312-thread-2) [n:127.0.0.1:40565_solr     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 3575095 INFO  (jetty-launcher-17312-thread-5) [n:127.0.0.1:38281_solr     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 3575096 INFO  (zkConnectionManagerCallback-17365-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3575096 INFO  (jetty-launcher-17312-thread-2) [n:127.0.0.1:40565_solr     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 3575096 INFO  (zkConnectionManagerCallback-17367-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3575096 INFO  (jetty-launcher-17312-thread-5) [n:127.0.0.1:38281_solr     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 3575174 INFO  (jetty-launcher-17312-thread-4) [n:127.0.0.1:46631_solr     ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:46631_solr
   [junit4]   2> 3575175 INFO  (jetty-launcher-17312-thread-4) [n:127.0.0.1:46631_solr     ] o.a.s.c.Overseer Overseer (id=72837869619380238-127.0.0.1:46631_solr-n_0000000000) starting
   [junit4]   2> 3575181 INFO  (jetty-launcher-17312-thread-4) [n:127.0.0.1:46631_solr     ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:46631_solr
   [junit4]   2> 3575181 INFO  (OverseerStateUpdate-72837869619380238-127.0.0.1:46631_solr-n_0000000000) [n:127.0.0.1:46631_solr     ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:46631_solr
   [junit4]   2> 3575184 INFO  (OverseerStateUpdate-72837869619380238-127.0.0.1:46631_solr-n_0000000000) [n:127.0.0.1:46631_solr     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 3575184 DEBUG (OverseerAutoScalingTriggerThread-72837869619380238-127.0.0.1:46631_solr-n_0000000000) [     ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
   [junit4]   2> 3575189 DEBUG (OverseerAutoScalingTriggerThread-72837869619380238-127.0.0.1:46631_solr-n_0000000000) [     ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
   [junit4]   2> 3575190 DEBUG (OverseerAutoScalingTriggerThread-72837869619380238-127.0.0.1:46631_solr-n_0000000000) [     ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 0
   [junit4]   2> 3575190 DEBUG (OverseerAutoScalingTriggerThread-72837869619380238-127.0.0.1:46631_solr-n_0000000000) [     ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
   [junit4]   2> 3575191 INFO  (jetty-launcher-17312-thread-1) [n:127.0.0.1:46051_solr     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 3575193 INFO  (jetty-launcher-17312-thread-3) [n:127.0.0.1:40305_solr     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 3575194 INFO  (jetty-launcher-17312-thread-5) [n:127.0.0.1:38281_solr     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 3575194 INFO  (jetty-launcher-17312-thread-4) [n:127.0.0.1:46631_solr     ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 3575195 DEBUG (OverseerAutoScalingTriggerThread-72837869619380238-127.0.0.1:46631_solr-n_0000000000) [     ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:46631_solr]
   [junit4]   2> 3575195 DEBUG (OverseerAutoScalingTriggerThread-72837869619380238-127.0.0.1:46631_solr-n_0000000000) [     ] o.a.s.c.a.OverseerTriggerThread -- deactivating old nodeLost / nodeAdded markers
   [junit4]   2> 3575195 INFO  (jetty-launcher-17312-thread-2) [n:127.0.0.1:40565_solr     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 3575196 DEBUG (OverseerAutoScalingTriggerThread-72837869619380238-127.0.0.1:46631_solr-n_0000000000) [     ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
   [junit4]   2> 3575196 DEBUG (ScheduledTrigger-16125-thread-2) [     ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
   [junit4]   2> 3575196 INFO  (jetty-launcher-17312-thread-1) [n:127.0.0.1:46051_solr     ] o.a.s.c.ZkController Publish node=127.0.0.1:46051_solr as DOWN
   [junit4]   2> 3575197 INFO  (jetty-launcher-17312-thread-3) [n:127.0.0.1:40305_solr     ] o.a.s.c.ZkController Publish node=127.0.0.1:40305_solr as DOWN
   [junit4]   2> 3575197 INFO  (jetty-launcher-17312-thread-1) [n:127.0.0.1:46051_solr     ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 3575197 INFO  (jetty-launcher-17312-thread-1) [n:127.0.0.1:46051_solr     ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:46051_solr
   [junit4]   2> 3575197 INFO  (jetty-launcher-17312-thread-3) [n:127.0.0.1:40305_solr     ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 3575197 INFO  (jetty-launcher-17312-thread-3) [n:127.0.0.1:40305_solr     ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40305_solr
   [junit4]   2> 3575202 INFO  (jetty-launcher-17312-thread-5) [n:127.0.0.1:38281_solr     ] o.a.s.c.ZkController Publish node=127.0.0.1:38281_solr as DOWN
   [junit4]   2> 3575202 INFO  (jetty-launcher-17312-thread-2) [n:127.0.0.1:40565_solr     ] o.a.s.c.ZkController Publish node=127.0.0.1:40565_solr as DOWN
   [junit4]   2> 3575203 INFO  (jetty-launcher-17312-thread-5) [n:127.0.0.1:38281_solr     ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 3575203 INFO  (jetty-launcher-17312-thread-5) [n:127.0.0.1:38281_solr     ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38281_solr
   [junit4]   2> 3575203 INFO  (jetty-launcher-17312-thread-2) [n:127.0.0.1:40565_solr     ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 3575203 INFO  (jetty-launcher-17312-thread-2) [n:127.0.0.1:40565_solr     ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40565_solr
   [junit4]   2> 3575219 INFO  (jetty-launcher-17312-thread-1) [n:127.0.0.1:46051_solr     ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 3575220 INFO  (jetty-launcher-17312-thread-3) [n:127.0.0.1:40305_solr     ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 3575220 INFO  (jetty-launcher-17312-thread-5) [n:127.0.0.1:38281_solr     ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 3575222 INFO  (zkCallback-17364-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (5)
   [junit4]   2> 3575227 INFO  (jetty-launcher-17312-thread-2) [n:127.0.0.1:40565_solr     ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 3575234 INFO  (zkCallback-17366-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (5)
   [junit4]   2> 3575234 INFO  (zkCallback-17352-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (5)
   [junit4]   2> 3575233 INFO  (zkCallback-17362-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (5)
   [junit4]   2> 3575234 INFO  (jetty-launcher-17312-thread-4) [n:127.0.0.1:46631_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_46631.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@102814d4
   [junit4]   2> 3575238 INFO  (zkCallback-17360-thread-2) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (5)
   [junit4]   2> 3575275 INFO  (jetty-launcher-17312-thread-4) [n:127.0.0.1:46631_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_46631.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@102814d4
   [junit4]   2> 3575275 INFO  (jetty-launcher-17312-thread-4) [n:127.0.0.1:46631_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_46631.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@102814d4
   [junit4]   2> 3575278 INFO  (jetty-launcher-17312-thread-4) [n:127.0.0.1:46631_solr     ] o.a.s.p.PackageLoader /packages.json updated to version -1
   [junit4]   2> 3575279 INFO  (jetty-launcher-17312-thread-4) [n:127.0.0.1:46631_solr     ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.autoscaling.SearchRateTriggerIntegrationTest_23CA321706FE3269-001/tempDir-001/node4/.
   [junit4]   2> 3575287 INFO  (jetty-launcher-17312-thread-3) [n:127.0.0.1:40305_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40305.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@102814d4
   [junit4]   2> 3575289 INFO  (jetty-launcher-17312-thread-1) [n:127.0.0.1:46051_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_46051.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@102814d4
   [junit4]   2> 3575290 INFO  (jetty-launcher-17312-thread-2) [n:127.0.0.1:40565_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40565.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@102814d4
   [junit4]   2> 3575293 INFO  (jetty-launcher-17312-thread-5) [n:127.0.0.1:38281_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38281.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@102814d4
   [junit4]   2> 3575313 INFO  (jetty-launcher-17312-thread-3) [n:127.0.0.1:40305_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40305.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@102814d4
   [junit4]   2> 3575313 INFO  (jetty-launcher-17312-thread-3) [n:127.0.0.1:40305_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40305.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@102814d4
   [junit4]   2> 3575314 INFO  (jetty-launcher-17312-thread-1) [n:127.0.0.1:46051_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_46051.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@102814d4
   [junit4]   2> 3575314 INFO  (jetty-launcher-17312-thread-1) [n:127.0.0.1:46051_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_46051.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@102814d4
   [junit4]   2> 3575316 INFO  (jetty-launcher-17312-thread-3) [n:127.0.0.1:40305_solr     ] o.a.s.p.PackageLoader /packages.json updated to version -1
   [junit4]   2> 3575316 INFO  (jetty-launcher-17312-thread-1) [n:127.0.0.1:46051_solr     ] o.a.s.p.PackageLoader /packages.json updated to version -1
   [junit4]   2> 3575317 INFO  (jetty-launcher-17312-thread-3) [n:127.0.0.1:40305_solr     ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.autoscaling.SearchRateTriggerIntegrationTest_23CA321706FE3269-001/tempDir-001/node3/.
   [junit4]   2> 3575318 INFO  (jetty-launcher-17312-thread-2) [n:127.0.0.1:40565_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40565.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@102814d4
   [junit4]   2> 3575318 INFO  (jetty-launcher-17312-thread-2) [n:127.0.0.1:40565_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40565.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@102814d4
   [junit4]   2> 3575318 INFO  (jetty-launcher-17312-thread-1) [n:127.0.0.1:46051_solr     ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.autoscaling.SearchRateTriggerIntegrationTest_23CA321706FE3269-001/tempDir-001/node1/.
   [junit4]   2> 3575319 INFO  (jetty-launcher-17312-thread-2) [n:127.0.0.1:40565_solr     ] o.a.s.p.PackageLoader /packages.json updated to version -1
   [junit4]   2> 3575319 INFO  (jetty-launcher-17312-thread-2) [n:127.0.0.1:40565_solr     ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.autoscaling.SearchRateTriggerIntegrationTest_23CA321706FE3269-001/tempDir-001/node5/.
   [junit4]   2> 3575320 INFO  (jetty-launcher-17312-thread-5) [n:127.0.0.1:38281_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38281.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@102814d4
   [junit4]   2> 3575320 INFO  (jetty-launcher-17312-thread-5) [n:127.0.0.1:38281_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38281.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@102814d4
   [junit4]   2> 3575321 INFO  (jetty-launcher-17312-thread-5) [n:127.0.0.1:38281_solr     ] o.a.s.p.PackageLoader /packages.json updated to version -1
   [junit4]   2> 3575321 INFO  (jetty-launcher-17312-thread-5) [n:127.0.0.1:38281_solr     ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.autoscaling.SearchRateTriggerIntegrationTest_23CA321706FE3269-001/tempDir-001/node2/.
   [junit4]   2> 3575488 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[23CA321706FE3269]-worker) [     ] o.a.s.c.MiniSolrCloudCluster waitForAllNodes: numServers=5
   [junit4]   2> 3575489 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[23CA321706FE3269]-worker) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 3575489 INFO  (zkConnectionManagerCallback-17392-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3575489 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[23CA321706FE3269]-worker) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 3575490 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[23CA321706FE3269]-worker) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (5)
   [junit4]   2> 3575491 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[23CA321706FE3269]-worker) [     ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:38317/solr ready
   [junit4]   2> 3575495 INFO  (qtp367098529-64207) [n:127.0.0.1:38281_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/autoscaling params={wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 3575496 INFO  (qtp367098529-64211) [n:127.0.0.1:38281_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 3575498 INFO  (qtp2115974359-64176) [n:127.0.0.1:40305_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 3575499 INFO  (qtp1435227235-64186) [n:127.0.0.1:40565_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 3575500 INFO  (qtp1026160353-64201) [n:127.0.0.1:46051_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 3575501 INFO  (qtp1108765706-64196) [n:127.0.0.1:46631_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 3575501 DEBUG (qtp367098529-64209) [n:127.0.0.1:38281_solr     ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 3575501 INFO  (qtp367098529-64209) [n:127.0.0.1:38281_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/autoscaling params={wt=javabin&version=2} status=0 QTime=6
   [junit4] IGNOR/A 0.00s J2 | SearchRateTriggerIntegrationTest.testDeleteNode
   [junit4]    > Assumption #1: 'awaitsfix' test group is disabled (@AwaitsFix(bugUrl=https://issues.apache.org/jira/browse/SOLR-13163))
   [junit4]   2> 3575508 DEBUG (zkCallback-17360-thread-2) [     ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2
   [junit4]   2> 3575513 DEBUG (OverseerAutoScalingTriggerThread-72837869619380238-127.0.0.1:46631_solr-n_0000000000) [     ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
   [junit4]   2> 3575534 DEBUG (OverseerAutoScalingTriggerThread-72837869619380238-127.0.0.1:46631_solr-n_0000000000) [     ] o.a.s.c.a.OverseerTriggerThread -- deactivating old nodeLost / nodeAdded markers
   [junit4]   2> 3575541 DEBUG (OverseerAutoScalingTriggerThread-72837869619380238-127.0.0.1:46631_solr-n_0000000000) [     ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
   [junit4]   2> 3575620 INFO  (TEST-SearchRateTriggerIntegrationTest.testBelowSearchRate-seed#[23CA321706FE3269]) [     ] o.a.s.SolrTestCaseJ4 ###Starting testBelowSearchRate
   [junit4]   2> 3575622 INFO  (TEST-SearchRateTriggerIntegrationTest.testBelowSearchRate-seed#[23CA321706FE3269]) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (5)
   [junit4]   2> 3575624 INFO  (TEST-SearchRateTriggerIntegrationTest.testBelowSearchRate-seed#[23CA321706FE3269]) [     ] o.a.s.c.a.SearchRateTriggerIntegrationTest /autoscaling.json reset, new znode version 3
   [junit4]   2> 3575624 DEBUG (zkCallback-17360-thread-2) [     ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 3
   [junit4]   2> 3575624 DEBUG (OverseerAutoScalingTriggerThread-72837869619380238-127.0.0.1:46631_solr-n_0000000000) [     ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
   [junit4]   2> 3575626 DEBUG (OverseerAutoScalingTriggerThread-72837869619380238-127.0.0.1:46631_solr-n_0000000000) [     ] o.a.s.c.a.OverseerTriggerThread -- deactivating old nodeLost / nodeAdded markers
   [junit4]   2> 3575627 DEBUG (OverseerAutoScalingTriggerThread-72837869619380238-127.0.0.1:46631_solr-n_0000000000) [     ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
   [junit4]   2> 3575627 INFO  (qtp1435227235-64191) [n:127.0.0.1:40565_solr     ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf&name=belowRate_collection&nrtReplicas=2&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 3575629 INFO  (OverseerThreadFactory-16127-thread-1-processing-n:127.0.0.1:46631_solr) [n:127.0.0.1:46631_solr     ] o.a.s.c.a.c.CreateCollectionCmd Create collection belowRate_collection
   [junit4]   2> 3575736 INFO  (OverseerStateUpdate-72837869619380238-127.0.0.1:46631_solr-n_0000000000) [n:127.0.0.1:46631_solr     ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"belowRate_collection",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"belowRate_collection_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:46631/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 3575738 INFO  (OverseerStateUpdate-72837869619380238-127.0.0.1:46631_solr-n_0000000000) [n:127.0.0.1:46631_solr     ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"belowRate_collection",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"belowRate_collection_shard1_replica_n2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:40305/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 3575941 INFO  (qtp1108765706-64205) [n:127.0.0.1:46631_solr    x:belowRate_collection_shard1_replica_n1 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=belowRate_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=belowRate_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 3575941 INFO  (qtp1108765706-64205) [n:127.0.0.1:46631_solr    x:belowRate_collection_shard1_replica_n1 ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 3575941 INFO  (qtp2115974359-64178) [n:127.0.0.1:40305_solr    x:belowRate_collection_shard1_replica_n2 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=belowRate_collection_shard1_replica_n2&action=CREATE&numShards=1&collection=belowRate_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 3576951 INFO  (qtp1108765706-64205) [n:127.0.0.1:46631_solr c:belowRate_collection s:shard1 r:core_node3 x:belowRate_collection_shard1_replica_n1 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.4.0
   [junit4]   2> 3576951 INFO  (qtp2115974359-64178) [n:127.0.0.1:40305_solr c:belowRate_collection s:shard1 r:core_node4 x:belowRate_collection_shard1_replica_n2 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.4.0
   [junit4]   2> 3576959 INFO  (qtp1108765706-64205) [n:127.0.0.1:46631_solr c:belowRate_collection s:shard1 r:core_node3 x:belowRate_collection_shard1_replica_n1 ] o.a.s.s.IndexSchema [belowRate_collection_shard1_replica_n1] Schema name=minimal
   [junit4]   2> 3576959 INFO  (qtp2115974359-64178) [n:127.0.0.1:40305_solr c:belowRate_collection s:shard1 r:core_node4 x:belowRate_collection_shard1_replica_n2 ] o.a.s.s.IndexSchema [belowRate_collection_shard1_replica_n2] Schema name=minimal
   [junit4]   2> 3576961 INFO  (qtp1108765706-64205) [n:127.0.0.1:46631_solr c:belowRate_collection s:shard1 r:core_node3 x:belowRate_collection_shard1_replica_n1 ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 3576961 INFO  (qtp1108765706-64205) [n:127.0.0.1:46631_solr c:belowRate_collection s:shard1 r:core_node3 x:belowRate_collection_shard1_replica_n1 ] o.a.s.c.CoreContainer Creating SolrCore 'belowRate_collection_shard1_replica_n1' using configuration from collection belowRate_collection, trusted=true
   [junit4]   2> 3576961 INFO  (qtp2115974359-64178) [n:127.0.0.1:40305_solr c:belowRate_collection s:shard1 r:core_node4 x:belowRate_collection_shard1_replica_n2 ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 3576961 INFO  (qtp2115974359-64178) [n:127.0.0.1:40305_solr c:belowRate_collection s:shard1 r:core_node4 x:belowRate_collection_shard1_replica_n2 ] o.a.s.c.CoreContainer Creating SolrCore 'belowRate_collection_shard1_replica_n2' using configuration from collection belowRate_collection, trusted=true
   [junit4]   2> 3576961 INFO  (qtp1108765706-64205) [n:127.0.0.1:46631_solr c:belowRate_collection s:shard1 r:core_node3 x:belowRate_collection_shard1_replica_n1 ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_46631.solr.core.belowRate_collection.shard1.replica_n1' (registry 'solr.core.belowRate_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@102814d4
   [junit4]   2> 3576961 INFO  (qtp2115974359-64178) [n:127.0.0.1:40305_solr c:belowRate_collection s:shard1 r:core_node4 x:belowRate_collection_shard1_replica_n2 ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40305.solr.core.belowRate_collection.shard1.replica_n2' (registry 'solr.core.belowRate_collection.shard1.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@102814d4
   [junit4]   2> 3576962 INFO  (qtp1108765706-64205) [n:127.0.0.1:46631_solr c:belowRate_collection s:shard1 r:core_node3 x:belowRate_collection_shard1_replica_n1 ] o.a.s.c.SolrCore [[belowRate_collection_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.autoscaling.SearchRateTriggerIntegrationTest_23CA321706FE3269-001/tempDir-001/node4/belowRate_collection_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.autoscaling.SearchRateTriggerIntegrationTest_23CA321706FE3269-001/tempDir-001/node4/./belowRate_collection_shard1_replica_n1/data/]
   [junit4]   2> 3576962 INFO  (qtp2115974359-64178) [n:127.0.0.1:40305_solr c:belowRate_collection s:shard1 r:core_node4 x:belowRate_collection_shard1_replica_n2 ] o.a.s.c.SolrCore [[belowRate_collection_shard1_replica_n2] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.autoscaling.SearchRateTriggerIntegrationTest_23CA321706FE3269-001/tempDir-001/node3/belowRate_collection_shard1_replica_n2], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.autoscaling.SearchRateTriggerIntegrationTest_23CA321706FE3269-001/tempDir-001/node3/./belowRate_collection_shard1_replica_n2/data/]
   [junit4]   2> 3577009 INFO  (qtp1108765706-64205) [n:127.0.0.1:46631_solr c:belowRate_collection s:shard1 r:core_node3 x:belowRate_collection_shard1_replica_n1 ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 3577009 INFO  (qtp1108765706-64205) [n:127.0.0.1:46631_solr c:belowRate_collection s:shard1 r:core_node3 x:belowRate_collection_shard1_replica_n1 ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 3577009 INFO  (qtp2115974359-64178) [n:127.0.0.1:40305_solr c:belowRate_collection s:shard1 r:core_node4 x:belowRate_collection_shard1_replica_n2 ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 3577009 INFO  (qtp2115974359-64178) [n:127.0.0.1:40305_solr c:belowRate_collection s:shard1 r:core_node4 x:belowRate_collection_shard1_replica_n2 ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 3577010 INFO  (qtp2115974359-64178) [n:127.0.0.1:40305_solr c:belowRate_collection s:shard1 r:core_node4 x:belowRate_collection_shard1_replica_n2 ] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 3577010 INFO  (qtp2115974359-64178) [n:127.0.0.1:40305_solr c:belowRate_collection s:shard1 r:core_node4 x:belowRate_collection_shard1_replica_n2 ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 3577010 INFO  (qtp1108765706-64205) [n:127.0.0.1:46631_solr c:belowRate_collection s:shard1 r:core_node3 x:belowRate_collection_shard1_replica_n1 ] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 3577010 INFO  (qtp1108765706-64205) [n:127.0.0.1:46631_solr c:belowRate_collection s:shard1 r:core_node3 x:belowRate_collection_shard1_replica_n1 ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 3577012 INFO  (qtp1108765706-64205) [n:127.0.0.1:46631_solr c:belowRate_collection s:shard1 r:core_node3 x:belowRate_collection_shard1_replica_n1 ] o.a.s.s.SolrIndexSearcher Opening [Searcher@5a383639[belowRate_collection_shard1_replica_n1] main]
   [junit4]   2> 3577012 INFO  (qtp2115974359-64178) [n:127.0.0.1:40305_solr c:belowRate_collection s:shard1 r:core_node4 x:belowRate_collection_shard1_replica_n2 ] o.a.s.s.SolrIndexSearcher Opening [Searcher@5cf0c4bc[belowRate_collection_shard1_replica_n2] main]
   [junit4]   2> 3577013 INFO  (qtp2115974359-64178) [n:127.0.0.1:40305_solr c:belowRate_collection s:shard1 r:core_node4 x:belowRate_collection_shard1_replica_n2 ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 3577013 INFO  (qtp1108765706-64205) [n:127.0.0.1:46631_solr c:belowRate_collection s:shard1 r:core_node3 x:belowRate_collection_shard1_replica_n1 ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 3577014 INFO  (qtp1108765706-64205) [n:127.0.0.1:46631_solr c:belowRate_collection s:shard1 r:core_node3 x:belowRate_collection_shard1_replica_n1 ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 3577014 INFO  (qtp2115974359-64178) [n:127.0.0.1:40305_solr c:belowRate_collection s:shard1 r:core_node4 x:belowRate_collection_shard1_replica_n2 ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 3577014 INFO  (qtp1108765706-64205) [n:127.0.0.1:46631_solr c:belowRate_collection s:shard1 r:core_node3 x:belowRate_collection_shard1_replica_n1 ] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 3577014 INFO  (qtp2115974359-64178) [n:127.0.0.1:40305_solr c:belowRate_collection s:shard1 r:core_node4 x:belowRate_collection_shard1_replica_n2 ] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 3577014 INFO  (qtp1108765706-64205) [n:127.0.0.1:46631_solr c:belowRate_collection s:shard1 r:core_node3 x:belowRate_collection_shard1_replica_n1 ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1652015203640934400
   [junit4]   2> 3577014 INFO  (qtp2115974359-64178) [n:127.0.0.1:40305_solr c:belowRate_collection s:shard1 r:core_node4 x:belowRate_collection_shard1_replica_n2 ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1652015203640934400
   [junit4]   2> 3577016 INFO  (searcherExecutor-16148-thread-1-processing-n:127.0.0.1:46631_solr x:belowRate_collection_shard1_replica_n1 c:belowRate_collection s:shard1 r:core_node3) [n:127.0.0.1:46631_solr c:belowRate_collection s:shard1 r:core_node3 x:belowRate_collection_shard1_replica_n1 ] o.a.s.c.SolrCore [belowRate_collection_shard1_replica_n1] Registered new searcher Searcher@5a383639[belowRate_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 3577016 INFO  (searcherExecutor-16149-thread-1-processing-n:127.0.0.1:40305_solr x:belowRate_collection_shard1_replica_n2 c:belowRate_collection s:shard1 r:core_node4) [n:127.0.0.1:40305_solr c:belowRate_collection s:shard1 r:core_node4 x:belowRate_collection_shard1_replica_n2 ] o.a.s.c.SolrCore [belowRate_collection_shard1_replica_n2] Registered new searcher Searcher@5cf0c4bc[belowRate_collection_shard1_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 3577018 INFO  (qtp1108765706-64205) [n:127.0.0.1:46631_solr c:belowRate_collection s:shard1 r:core_node3 x:belowRate_collection_shard1_replica_n1 ] o.a.s.c.ZkShardTerms Successful update of terms at /collections/belowRate_collection/terms/shard1 to Terms{values={core_node3=0}, version=0}
   [junit4]   2> 3577018 INFO  (qtp1108765706-64205) [n:127.0.0.1:46631_solr c:belowRate_collection s:shard1 r:core_node3 x:belowRate_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/belowRate_collection/leaders/shard1
   [junit4]   2> 3577018 INFO  (qtp2115974359-64178) [n:127.0.0.1:40305_solr c:belowRate_collection s:shard1 r:core_node4 x:belowRate_collection_shard1_replica_n2 ] o.a.s.c.ZkShardTerms Failed to save terms, version is not a match, retrying
   [junit4]   2> 3577019 INFO  (qtp2115974359-64178) [n:127.0.0.1:40305_solr c:belowRate_collection s:shard1 r:core_node4 x:belowRate_collection_shard1_replica_n2 ] o.a.s.c.ZkShardTerms Successful update of terms at /collections/belowRate_collection/terms/shard1 to Terms{values={core_node3=0, core_node4=0}, version=1}
   [junit4]   2> 3577019 INFO  (qtp2115974359-64178) [n:127.0.0.1:40305_solr c:belowRate_collection s:shard1 r:core_node4 x:belowRate_collection_shard1_replica_n2 ] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/belowRate_collection/leaders/shard1
   [junit4]   2> 3577020 INFO  (qtp1108765706-64205) [n:127.0.0.1:46631_solr c:belowRate_collection s:shard1 r:core_node3 x:belowRate_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 3577020 INFO  (qtp1108765706-64205) [n:127.0.0.1:46631_solr c:belowRate_collection s:shard1 r:core_node3 x:belowRate_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 3577020 INFO  (qtp1108765706-64205) [n:127.0.0.1:46631_solr c:belowRate_collection s:shard1 r:core_node3 x:belowRate_collection_shard1_replica_n1 ] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:46631/solr/belowRate_collection_shard1_replica_n1/
   [junit4]   2> 3577021 INFO  (qtp1108765706-64205) [n:127.0.0.1:46631_solr c:belowRate_collection s:shard1 r:core_node3 x:belowRate_collection_shard1_replica_n1 ] o.a.s.u.PeerSync PeerSync: core=belowRate_collection_shard1_replica_n1 url=http://127.0.0.1:46631/solr START replicas=[http://127.0.0.1:40305/solr/belowRate_collection_shard1_replica_n2/] nUpdates=100
   [junit4]   2> 3577021 INFO  (qtp1108765706-64205) [n:127.0.0.1:46631_solr c:belowRate_collection s:shard1 r:core_node3 x:belowRate_collection_shard1_replica_n1 ] o.a.s.u.PeerSync PeerSync: core=belowRate_collection_shard1_replica_n1 url=http://127.0.0.1:46631/solr DONE.  We have no versions.  sync failed.
   [junit4]   2> 3577022 INFO  (qtp2115974359-64175) [n:127.0.0.1:40305_solr c:belowRate_collection s:shard1 r:core_node4 x:belowRate_collection_shard1_replica_n2 ] o.a.s.c.S.Request [belowRate_collection_shard1_replica_n2]  webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 3577022 INFO  (qtp1108765706-64205) [n:127.0.0.1:46631_solr c:belowRate_collection s:shard1 r:core_node3 x:belowRate_collection_shard1_replica_n1 ] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
   [junit4]   2> 3577022 INFO  (qtp1108765706-64205) [n:127.0.0.1:46631_solr c:belowRate_collection s:shard1 r:core_node3 x:belowRate_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 3577022 INFO  (qtp1108765706-64205) [n:127.0.0.1:46631_solr c:belowRate_collection s:shard1 r:core_node3 x:belowRate_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/belowRate_collection/leaders/shard1/leader after winning as /collections/belowRate_collection/leader_elect/shard1/election/72837869619380238-core_node3-n_0000000000
   [junit4]   2> 3577023 INFO  (qtp1108765706-64205) [n:127.0.0.1:46631_solr c:belowRate_collection s:shard1 r:core_node3 x:belowRate_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:46631/solr/belowRate_collection_shard1_replica_n1/ shard1
   [junit4]   2> 3577124 INFO  (zkCallback-17360-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/belowRate_collection/state.json] for collection [belowRate_collection] has occurred - updating... (live nodes size: [5])
   [junit4]   2> 3577124 INFO  (zkCallback-17360-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/belowRate_collection/state.json] for collection [belowRate_collection] has occurred - updating... (live nodes size: [5])
   [junit4]   2> 3577125 INFO  (qtp1108765706-64205) [n:127.0.0.1:46631_solr c:belowRate_collection s:shard1 r:core_node3 x:belowRate_collection_shard1_replica_n1 ] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 3577126 INFO  (qtp1108765706-64205) [n:127.0.0.1:46631_solr c:belowRate_collection s:shard1 r:core_node3 x:belowRate_collection_shard1_replica_n1 ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=belowRate_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=belowRate_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1186
   [junit4]   2> 3577227 INFO  (zkCallback-17360-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/belowRate_collection/state.json] for collection [belowRate_collection] has occurred - updating... (live nodes size: [5])
   [junit4]   2> 3577227 INFO  (zkCallback-17360-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/belowRate_collection/state.json] for collection [belowRate_collection] has occurred - updating... (live nodes size: [5])
   [junit4]   2> 3577227 INFO  (zkCallback-17360-thread-3) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/belowRate_collection/state.json] for collection [belowRate_collection] has occurred - updating... (live nodes size: [5])
   [junit4]   2> 3578023 INFO  (qtp2115974359-64178) [n:127.0.0.1:40305_solr c:belowRate_collection s:shard1 r:core_node4 x:belowRate_collection_shard1_replica_n2 ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=belowRate_collection_shard1_replica_n2&action=CREATE&numShards=1&collection=belowRate_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2081
   [junit4]   2> 3578026 INFO  (qtp1435227235-64191) [n:127.0.0.1:40565_solr     ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
   [junit4]   2> 3578123 INFO  (zkCallback-17364-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/belowRate_collection/state.json] for collection [belowRate_collection] has occurred - updating... (live nodes size: [5])
   [junit4]   2> 3578123 INFO  (zkCallback-17360-thread-3) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/belowRate_collection/state.json] for collection [belowRate_collection] has occurred - updating... (live nodes size: [5])
   [junit4]   2> 3578123 INFO  (zkCallback-17360-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/belowRate_collection/state.json] for collection [belowRate_collection] has occurred - updating... (live nodes size: [5])
   [junit4]   2> 3578123 INFO  (zkCallback-17362-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/belowRate_collection/state.json] for collection [belowRate_collection] has occurred - updating... (live nodes size: [5])
   [junit4]   2> 3578123 INFO  (zkCallback-17360-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/belowRate_collection/state.json] for collection [belowRate_collection] has occurred - updating... (live nodes size: [5])
   [junit4]   2> 3578123 INFO  (zkCallback-17362-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/belowRate_collection/state.json] for collection [belowRate_collection] has occurred - updating... (live nodes size: [5])
   [junit4]   2> 3578123 INFO  (qtp1435227235-64191) [n:127.0.0.1:40565_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf&name=belowRate_collection&nrtReplicas=2&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=2496
   [junit4]   2> 3578124 INFO  (qtp1435227235-64200) [n:127.0.0.1:40565_solr     ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params action=ADDREPLICA&collection=belowRate_collection&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 3578129 INFO  (OverseerCollectionConfigSetProcessor-72837869619380238-127.0.0.1:46631_solr-n_0000000000) [n:127.0.0.1:46631_solr     ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 3578129 INFO  (OverseerThreadFactory-16127-thread-2-processing-n:127.0.0.1:46631_solr) [n:127.0.0.1:46631_solr c:belowRate_collection s:shard1   ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:40565_solr for creating new replica of shard shard1 for collection belowRate_collection
   [junit4]   2> 3578130 INFO  (OverseerThreadFactory-16127-thread-2-processing-n:127.0.0.1:46631_solr) [n:127.0.0.1:46631_solr c:belowRate_collection s:shard1   ] o.a.s.c.a.c.AddReplicaCmd Returning CreateReplica command.
   [junit4]   2> 3578131 INFO  (OverseerStateUpdate-72837869619380238-127.0.0.1:46631_solr-n_0000000000) [n:127.0.0.1:46631_solr     ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"addreplica",
   [junit4]   2>   "collection":"belowRate_collection",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"belowRate_collection_shard1_replica_n5",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:40565/solr",
   [junit4]   2>   "node_name":"127.0.0.1:40565_solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 3578233 INFO  (zkCallback-17360-thread-3) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/belowRate_collection/state.json] for collection [belowRate_collection] has occurred - updating... (live nodes size: [5])
   [junit4]   2> 3578233 INFO  (zkCallback-17360-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/belowRate_collection/state.json] for collection [belowRate_collection] has occurred - updating... (live nodes size: [5])
   [junit4]   2> 3578233 INFO  (zkCallback-17362-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/belowRate_collection/state.json] for collection [belowRate_collection] has occurred - updating... (live nodes size: [5])
   [junit4]   2> 3578233 INFO  (zkCallback-17362-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/belowRate_collection/state.json] for collection [belowRate_collection] has occurred - updating... (live nodes size: [5])
   [junit4]   2> 3578233 INFO  (zkCallback-17360-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/belowRate_collection/state.json] for collection [belowRate_collection] has occurred - updating... (live nodes size: [5])
   [junit4]   2> 3578334 INFO  (qtp1435227235-64187) [n:127.0.0.1:40565_solr    x:belowRate_collection_shard1_replica_n5 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node6&collection.configName=conf&name=belowRate_collection_shard1_replica_n5&action=CREATE&collection=belowRate_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 3578343 INFO  (qtp1435227235-64187) [n:127.0.0.1:40565_solr c:belowRate_collection s:shard1 r:core_node6 x:belowRate_collection_shard1_replica_n5 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.4.0
   [junit4]   2> 3578350 INFO  (qtp1435227235-64187) [n:127.0.0.1:40565_solr c:belowRate_collection s:shard1 r:core_node6 x:belowRate_collection_shard1_replica_n5 ] o.a.s.s.IndexSchema [belowRate_collection_shard1_replica_n5] Schema name=minimal
   [junit4]   2> 3578353 INFO  (qtp1435227235-64187) [n:127.0.0.1:40565_solr c:belowRate_collection s:shard1 r:core_node6 x:belowRate_collection_shard1_replica_n5 ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 3578353 INFO  (qtp1435227235-64187) [n:127.0.0.1:40565_solr c:belowRate_collection s:shard1 r:core_node6 x:belowRate_collection_shard1_replica_n5 ] o.a.s.c.CoreContainer Creating SolrCore 'belowRate_collection_shard1_replica_n5' using configuration from collection belowRate_collection, trusted=true
   [junit4]   2> 3578353 INFO  (qtp1435227235-64187) [n:127.0.0.1:40565_solr c:belowRate_collection s:shard1 r:core_node6 x:belowRate_collection_shard1_replica_n5 ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40565.solr.core.belowRate_collection.shard1.replica_n5' (registry 'solr.core.belowRate_collection.shard1.replica_n5') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@102814d4
   [junit4]   2> 3578353 INFO  (qtp1435227235-64187) [n:127.0.0.1:40565_solr c:belowRate_collection s:shard1 r:core_node6 x:belowRate_collection_shard1_replica_n5 ] o.a.s.c.SolrCore [[belowRate_collection_shard1_replica_n5] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.autoscaling.SearchRateTriggerIntegrationTest_23CA321706FE3269-001/tempDir-001/node5/belowRate_collection_shard1_replica_n5], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.autoscaling.SearchRateTriggerIntegrationTest_23CA321706FE3269-001/tempDir-001/node5/./belowRate_collection_shard1_replica_n5/data/]
   [junit4]   2> 3578387 INFO  (qtp1435227235-64187) [n:127.0.0.1:40565_solr c:belowRate_collection s:shard1 r:core_node6 x:belowRate_collection_shard1_replica_n5 ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 3578387 INFO  (qtp1435227235-64187) [n:127.0.0.1:40565_solr c:belowRate_collection s:shard1 r:core_node6 x:belowRate_collection_shard1_replica_n5 ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 3578388 INFO  (qtp1435227235-64187) [n:127.0.0.1:40565_solr c:belowRate_collection s:shard1 r:core_node6 x:belowRate_collection_shard1_replica_n5 ] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 3578389 INFO  (qtp1435227235-64187) [n:127.0.0.1:40565_solr c:belowRate_collection s:shard1 r:core_node6 x:belowRate_collection_shard1_replica_n5 ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 3578390 INFO  (qtp1435227235-64187) [n:127.0.0.1:40565_solr c:belowRate_collection s:shard1 r:core_node6 x:belowRate_collection_shard1_replica_n5 ] o.a.s.s.SolrIndexSearcher Opening [Searcher@cc4866f[belowRate_collection_shard1_replica_n5] main]
   [junit4]   2> 3578391 INFO  (qtp1435227235-64187) [n:127.0.0.1:40565_solr c:belowRate_collection s:shard1 r:core_node6 x:belowRate_collection_shard1_replica_n5 ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 3578391 INFO  (qtp1435227235-64187) [n:127.0.0.1:40565_solr c:belowRate_collection s:shard1 r:core_node6 x:belowRate_collection_shard1_replica_n5 ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 3578391 INFO  (qtp1435227235-64187) [n:127.0.0.1:40565_solr c:belowRate_collection s:shard1 r:core_node6 x:belowRate_collection_shard1_replica_n5 ] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 3578392 INFO  (qtp1435227235-64187) [n:127.0.0.1:40565_solr c:belowRate_collection s:shard1 r:core_node6 x:belowRate_collection_shard1_replica_n5 ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1652015205085872128
   [junit4]   2> 3578394 INFO  (searcherExecutor-16158-thread-1-processing-n:127.0.0.1:40565_solr x:belowRate_collection_shard1_replica_n5 c:belowRate_collection s:shard1 r:core_node6) [n:127.0.0.1:40565_solr c:belowRate_collection s:shard1 r:core_node6 x:belowRate_collection_shard1_replica_n5 ] o.a.s.c.SolrCore [belowRate_collection_shard1_replica_n5] Registered new searcher Searcher@cc4866f[belowRate_collection_shard1_replica_n5] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 3578395 INFO  (qtp1435227235-64187) [n:127.0.0.1:40565_solr c:belowRate_collection s:shard1 r:core_node6 x:belowRate_collection_shard1_replica_n5 ] o.a.s.c.ZkShardTerms Successful update of terms at /collections/belowRate_collection/terms/shard1 to Terms{values={core_node6=0, core_node3=0, core_node4=0}, version=2}
   [junit4]   2> 3578396 INFO  (qtp1435227235-64187) [n:127.0.0.1:40565_solr c:belowRate_collection s:shard1 r:core_node6 x:belowRate_collection_shard1_replica_n5 ] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/belowRate_collection/leaders/shard1
   [junit4]   2> 3578397 INFO  (qtp1435227235-64187) [n:127.0.0.1:40565_solr c:belowRate_collection s:shard1 r:core_node6 x:belowRate_collection_shard1_replica_n5 ] o.a.s.c.ZkController Core needs to recover:belowRate_collection_shard1_replica_n5
   [junit4]   2> 3578398 INFO  (updateExecutor-17347-thread-1-processing-n:127.0.0.1:40565_solr x:belowRate_collection_shard1_replica_n5 c:belowRate_collection s:shard1 r:core_node6) [n:127.0.0.1:40565_solr c:belowRate_collection s:shard1 r:core_node6 x:belowRate_collection_shard1_replica_n5 ] o.a.s.u.DefaultSolrCoreState Running recovery
   [junit4]   2> 3578398 INFO  (recoveryExecutor-17349-thread-1-processing-n:127.0.0.1:40565_solr x:belowRate_collection_shard1_replica_n5 c:belowRate_collection s:shard1 r:core_node6) [n:127.0.0.1:40565_solr c:belowRate_collection s:shard1 r:core_node6 x:belowRate_collection_shard1_replica_n5 ] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
   [junit4]   2> 3578398 INFO  (recoveryExecutor-17349-thread-1-processing-n:127.0.0.1:40565_solr x:belowRate_collection_shard1_replica_n5 c:belowRate_collection s:shard1 r:core_node6) [n:127.0.0.1:40565_solr c:belowRate_collection s:shard1 r:core_node6 x:belowRate_collection_shard1_replica_n5 ] o.a.s.c.RecoveryStrategy startupVersions is empty
   [junit4]   2> 3578399 INFO  (qtp1435227235-64187) [n:127.0.0.1:40565_solr c:belowRate_collection s:shard1 r:core_node6 x:belowRate_collection_shard1_replica_n5 ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node6&collection.configName=conf&name=belowRate_collection_shard1_replica_n5&action=CREATE&collection=belowRate_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=64
   [junit4]   2> 3578399 INFO  (qtp1108765706-64194) [n:127.0.0.1:46631_solr c:belowRate_collection s:shard1 r:core_node3 x:belowRate_collection_shard1_replica_n1 ] o.a.s.c.S.Request [belowRate_collection_shard1_replica_n1]  webapp=/solr path=/admin/ping params={wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 3578399 INFO  (qtp1108765706-64194) [n:127.0.0.1:46631_solr c:belowRate_collection s:shard1 r:core_node3 x:belowRate_collection_shard1_replica_n1 ] o.a.s.c.S.Request [belowRate_collection_shard1_replica_n1]  webapp=/solr path=/admin/ping params={wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 3578399 INFO  (recoveryExecutor-17349-thread-1-processing-n:127.0.0.1:40565_solr x:belowRate_collection_shard1_replica_n5 c:belowRate_collection s:shard1 r:core_node6) [n:127.0.0.1:40565_solr c:belowRate_collection s:shard1 r:core_node6 x:belowRate_collection_shard1_replica_n5 ] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[belowRate_collection_shard1_replica_n5]
   [junit4]   2> 3578399 INFO  (recoveryExecutor-17349-thread-1-processing-n:127.0.0.1:40565_solr x:belowRate_collection_shard1_replica_n5 c:belowRate_collection s:shard1 r:core_node6) [n:127.0.0.1:40565_solr c:belowRate_collection s:shard1 r:core_node6 x:belowRate_collection_shard1_replica_n5 ] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
   [junit4]   2> 3578399 INFO  (recoveryExecutor-17349-thread-1-processing-n:127.0.0.1:40565_solr x:belowRate_collection_shard1_replica_n5 c:belowRate_collection s:shard1 r:core_node6) [n:127.0.0.1:40565_solr c:belowRate_collection s:shard1 r:core_node6 x:belowRate_collection_shard1_replica_n5 ] o.a.s.c.RecoveryStrategy Publishing state of core [belowRate_collection_shard1_replica_n5] as recovering, leader is [http://127.0.0.1:46631/solr/belowRate_collection_shard1_replica_n1/] and I am [http://127.0.0.1:40565/solr/belowRate_collection_shard1_replica_n5/]
   [junit4]   2> 3578400 INFO  (recoveryExecutor-17349-thread-1-processing-n:127.0.0.1:40565_solr x:belowRate_collection_shard1_replica_n5 c:belowRate_collection s:shard1 r:core_node6) [n:127.0.0.1:40565_solr c:belowRate_collection s:shard1 r:core_node6 x:belowRate_collection_shard1_replica_n5 ] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:46631/solr]; [WaitForState: action=PREPRECOVERY&core=belowRate_collection_shard1_replica_n1&nodeName=127.0.0.1:40565_solr&coreNodeName=core_node6&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
   [junit4]   2> 3578400 INFO  (qtp1435227235-64200) [n:127.0.0.1:40565_solr c:belowRate_collection    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=ADDREPLICA&collection=belowRate_collection&shard=shard1&type=NRT&wt=javabin&version=2} status=0 QTime=276
   [junit4]   2> 3578400 INFO  (qtp1108765706-64196) [n:127.0.0.1:46631_solr    x:belowRate_collection_shard1_replica_n1 ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node6, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
   [junit4]   2> 3578400 INFO  (qtp1108765706-64196) [n:127.0.0.1:46631_solr    x:belowRate_collection_shard1_replica_n1 ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=belowRate_collection, shard=shard1, thisCore=belowRate_collection_shard1_replica_n1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:40565_solr, coreNodeName=core_node6, onlyIfActiveCheckResult=false, nodeProps: core_node6:{
   [junit4]   2>   "core":"belowRate_collection_shard1_replica_n5",
   [junit4]   2>   "base_url":"http://127.0.0.1:40565/solr",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "node_name":"127.0.0.1:40565_solr",
   [junit4]   2>   "type":"NRT"}
   [junit4]   2> 3578400 INFO  (qtp1108765706-64196) [n:127.0.0.1:46631_solr    x:belowRate_collection_shard1_replica_n1 ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=belowRate_collection, shard=shard1, thisCore=belowRate_collection_shard1_replica_n1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:40565_solr, coreNodeName=core_node6, onlyIfActiveCheckResult=false, nodeProps: core_node6:{
   [junit4]   2>   "core":"belowRate_collection_shard1_replica_n5",
   [junit4]   2>   "base_url":"http://127.0.0.1:40565/solr",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "node_name":"127.0.0.1:40565_solr",
   [junit4]   2>   "type":"NRT"}
   [junit4]   2> 3578400 INFO  (qtp1108765706-64196) [n:127.0.0.1:46631_solr    x:belowRate_collection_shard1_replica_n1 ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=belowRate_collection, shard=shard1, thisCore=belowRate_collection_shard1_replica_n1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:40565_solr, coreNodeName=core_node6, onlyIfActiveCheckResult=false, nodeProps: core_node6:{
   [junit4]   2>   "core":"belowRate_collection_shard1_replica_n5",
   [junit4]   2>   "base_url":"http://127.0.0.1:40565/solr",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "node_name":"127.0.0.1:4056

[...truncated too long message...]

2> 	27	/solr/collections/belowRate_collection/terms/shard1
   [junit4]   2> 	21	/solr/collections/aboveRate_collection/terms/shard1
   [junit4]   2> 	10	/solr/configs/conf
   [junit4]   2> 	8	/solr/clusterprops.json
   [junit4]   2> 	8	/solr/aliases.json
   [junit4]   2> 	5	/solr/packages.json
   [junit4]   2> 	5	/solr/security.json
   [junit4]   2> 	5	/solr/collections/aboveRate_collection/collectionprops.json
   [junit4]   2> 	4	/solr/collections/belowRate_collection/collectionprops.json
   [junit4]   2> 	3	/solr/collections/belowRate_collection/state.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	166	/solr/collections/aboveRate_collection/state.json
   [junit4]   2> 	144	/solr/collections/belowRate_collection/state.json
   [junit4]   2> 	20	/solr/autoscaling.json
   [junit4]   2> 	15	/solr/aliases.json
   [junit4]   2> 	8	/solr/clusterstate.json
   [junit4]   2> 	2	/solr/collections/belowRate_collection/leader_elect/shard1/election/72837869619380238-core_node3-n_0000000000
   [junit4]   2> 	2	/solr/collections/aboveRate_collection/leader_elect/shard1/election/72837869619380241-core_node6-n_0000000002
   [junit4]   2> 	2	/solr/collections/belowRate_collection/leader_elect/shard1/election/72837869619380239-core_node4-n_0000000001
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	85	/solr/overseer/queue
   [junit4]   2> 	35	/solr/overseer/collection-queue-work
   [junit4]   2> 	27	/solr/collections
   [junit4]   2> 	20	/solr/live_nodes
   [junit4]   2> 	6	/solr/autoscaling/events/search_rate_trigger2
   [junit4]   2> 	2	/solr/autoscaling/events/search_rate_trigger1
   [junit4]   2> 
   [junit4]   2> 3640856 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[23CA321706FE3269]-worker) [     ] o.a.s.c.ZkTestServer waitForServerDown: 127.0.0.1:38317
   [junit4]   2> 3640856 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[23CA321706FE3269]-worker) [     ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:38317
   [junit4]   2> 3640856 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[23CA321706FE3269]-worker) [     ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 38317
   [junit4]   2> 3640856 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[23CA321706FE3269]-worker) [     ] o.a.s.SolrTestCaseJ4 ------------------------------------------------------- Done waiting for tracked resources to be released
   [junit4]   2> Dec 04, 2019 6:35:34 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene84): {}, docValues:{}, maxPointsInLeafNode=1156, maxMBSortInHeap=5.0376712850069865, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@386d23e4), locale=es-EC, timezone=Africa/Dar_es_Salaam
   [junit4]   2> NOTE: Linux 4.15.0-54-generic amd64/Oracle Corporation 1.8.0_191 (64-bit)/cpus=4,threads=1,free=230319928,total=510132224
   [junit4]   2> NOTE: All tests run in this JVM: [DistributedMLTComponentTest, TestSolrCLIRunExample, CheckHdfsIndexTest, ResponseBuilderTest, DocValuesMissingTest, SpellCheckCollatorWithCollapseTest, TestMinMaxOnMultiValuedField, SolrMetricsIntegrationTest, ParsingFieldUpdateProcessorsTest, DistributedQueryElevationComponentTest, TestManagedSchemaThreadSafety, RollingRestartTest, AddSchemaFieldsUpdateProcessorFactoryTest, TestUninvertingReader, TaggerTest, TimeRoutedAliasUpdateProcessorTest, TestCoreContainer, TestExpandComponent, TestConfigSetsAPIZkFailure, FieldAnalysisRequestHandlerTest, TestFieldCollectionResource, TestChildDocTransformerHierarchy, TestNestedUpdateProcessor, OverseerModifyCollectionTest, PeerSyncWithBufferUpdatesTest, TestGraphMLResponseWriter, TestSortableTextField, TestSimpleTrackingShardHandler, TestCodecSupport, OutOfBoxZkACLAndCredentialsProvidersTest, TestTestInjection, ZkStateWriterTest, TestComplexPhraseQParserPlugin, NumericFieldsTest, PreAnalyzedFieldManagedSchemaCloudTest, PKIAuthenticationIntegrationTest, PeerSyncWithLeaderTest, RequiredFieldsTest, TermVectorComponentTest, TestExportTool, MigrateRouteKeyTest, HdfsChaosMonkeyNothingIsSafeTest, AtomicUpdatesTest, TestClusterProperties, NodeAddedTriggerIntegrationTest, TestRealTimeGet, TestCaffeineCache, TestDocSet, TestLeaderElectionWithEmptyReplica, TestXmlQParserPlugin, CollectionsAPIAsyncDistributedZkTest, BadComponentTest, HdfsUnloadDistributedZkTest, UnloadDistributedZkTest, TestCloudRecovery, TestStressCloudBlindAtomicUpdates, MoveReplicaHDFSTest, TestShortCircuitedRequests, TestManagedSynonymGraphFilterFactory, TriggerIntegrationTest, TestScoreJoinQPScore, TestDocBasedVersionConstraints, ExecutePlanActionTest, TestSimExecutePlanAction, XMLAtomicUpdateMultivalueTest, CoreAdminOperationTest, LegacyCloudClusterPropTest, TestPushWriter, TestLegacyBM25SimilarityFactory, TestTrackingShardHandlerFactory, SortSpecParsingTest, TestSchemaSimilarityResource, ConvertedLegacyTest, SolrJmxReporterTest, SolrCoreTest, DaemonStreamApiTest, TestSolrCloudWithDelegationTokens, TestZkAclsWithHadoopAuth, TestInPlaceUpdatesRequiredField, TestExactStatsCache, TestDefaultStatsCache, TestWriterPerf, SolrRequestParserTest, SpatialHeatmapFacetsTest, TestSimPolicyCloud, BooleanFieldTest, TestCustomStream, TestNumericRangeQuery32, SimpleCollectionCreateDeleteTest, RulesTest, TestUniqueKeyFieldResource, HLLUtilTest, LeaderElectionIntegrationTest, RegexBytesRefFilterTest, TestHighFrequencyDictionaryFactory, TestShardHandlerFactory, TestSchemaField, CreateRoutedAliasTest, MultiThreadedOCPTest, TestJsonFacetsWithNestedObjects, TestDocumentBuilder, TestDistributedGrouping, AutoscalingHistoryHandlerTest, ReindexCollectionTest, ScheduledMaintenanceTriggerTest, TestSolr4Spatial, DefaultValueUpdateProcessorTest, TestQueryTypes, TestTolerantSearch, SolrCloudExampleTest, TestPerFieldSimilarityWithDefaultOverride, TestRestoreCore, ZkControllerTest, TestPKIAuthenticationPlugin, TestZkChroot, TestSolrConfigHandlerCloud, TestFieldTypeResource, ClusterStateTest, MetricUtilsTest, TestLFUCache, FastVectorHighlighterTest, ClassificationUpdateProcessorIntegrationTest, DOMUtilTest, OutputWriterTest, TestDynamicFieldNamesIndexCorrectly, CdcrBootstrapTest, TestSolrDeletionPolicy1, TestPayloadCheckQParserPlugin, TestEmbeddedSolrServerConstructors, TestDistribDocBasedVersion, ShardsWhitelistTest, HttpPartitionTest, TestReload, DistribJoinFromCollectionTest, PeerSyncReplicationTest, TestTlogReplica, ExitableDirectoryReaderTest, PingRequestHandlerTest, TestSystemCollAutoCreate, TestIBSimilarityFactory, TestRetrieveFieldsOptimizer, TestConfigSetImmutable, DistributedVersionInfoTest, ForceLeaderWithTlogReplicasTest, LeaderElectionContextKeyTest, LeaderFailureAfterFreshStartTest, NestedShardedAtomicUpdateTest, NodeMutatorTest, OverriddenZkACLAndCredentialsProvidersTest, OverseerTaskQueueTest, RoutingToNodesWithPropertiesTest, SSLMigrationTest, SaslZkACLProviderTest, ShardRoutingCustomTest, SharedFSAutoReplicaFailoverTest, SolrXmlInZkTest, SplitShardTest, SystemCollectionCompatTest, TestAuthenticationFramework, TestCloudConsistency, TestCloudSearcherWarming, TestPullReplica, TestRandomFlRTGCloud, TestRequestForwarding, TestSegmentSorting, TestUtilizeNode, TestWaitForStateWithJettyShutdowns, TlogReplayBufferedWhileIndexingTest, TriLevelCompositeIdRoutingTest, ZkFailoverTest, ZkSolrClientTest, CustomCollectionTest, TestHdfsCloudBackupRestore, TestReplicaProperties, HdfsAutoAddReplicasIntegrationTest, NodeAddedTriggerTest, NodeLostTriggerIntegrationTest, NodeLostTriggerTest, NodeMarkersRegistrationTest, RestoreTriggerStateTest, ScheduledTriggerIntegrationTest, ScheduledTriggerTest, SearchRateTriggerIntegrationTest]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=SearchRateTriggerIntegrationTest -Dtests.seed=23CA321706FE3269 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/test-data/enwiki.random.lines.txt -Dtests.locale=es-EC -Dtests.timezone=Africa/Dar_es_Salaam -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J2 | SearchRateTriggerIntegrationTest (suite) <<<
   [junit4]    > Throwable #1: junit.framework.AssertionFailedError: Clean up static fields (in @AfterClass?) and null them, your test still has references to classes of which the sizes cannot be measured due to security restrictions or Java 9 module encapsulation:
   [junit4]    >   - private static org.apache.solr.client.solrj.cloud.SolrCloudManager org.apache.solr.cloud.autoscaling.SearchRateTriggerIntegrationTest.cloudManager
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([23CA321706FE3269]:0)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > Caused by: java.security.AccessControlException: access denied ("java.lang.RuntimePermission" "accessClassInPackage.sun.nio.fs")
   [junit4]    > 	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
   [junit4]    > 	at java.security.AccessController.checkPermission(AccessController.java:884)
   [junit4]    > 	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
   [junit4]    > 	at java.lang.SecurityManager.checkPackageAccess(SecurityManager.java:1564)
   [junit4]    > 	at java.lang.Class.checkPackageAccess(Class.java:2372)
   [junit4]    > 	at java.lang.Class.checkMemberAccess(Class.java:2351)
   [junit4]    > 	at java.lang.Class.getDeclaredFields(Class.java:1915)
   [junit4]    > 	at java.security.AccessController.doPrivileged(Native Method)
   [junit4]    > 	... 10 more
   [junit4] Completed [427/887 (1!)] on J2 in 67.56s, 3 tests, 1 failure, 1 skipped <<< FAILURES!

[...truncated 54428 lines...]

[JENKINS] Lucene-Solr-NightlyTests-8.x - Build # 287 - Still Unstable

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-8.x/287/

58 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.analytics.ExpressionFactoryTest

Error Message:
Clean up static fields (in @AfterClass?) and null them, your test still has references to classes of which the sizes cannot be measured due to security restrictions or Java 9 module encapsulation:   - private static org.apache.solr.schema.IndexSchema org.apache.solr.analytics.ExpressionFactoryTest.indexSchema

Stack Trace:
junit.framework.AssertionFailedError: Clean up static fields (in @AfterClass?) and null them, your test still has references to classes of which the sizes cannot be measured due to security restrictions or Java 9 module encapsulation:
  - private static org.apache.solr.schema.IndexSchema org.apache.solr.analytics.ExpressionFactoryTest.indexSchema
	at __randomizedtesting.SeedInfo.seed([2BB1AFE26D716F6F]:0)
	at com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule$1.afterAlways(StaticFieldsInvariantRule.java:146)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.security.AccessControlException: access denied ("java.lang.RuntimePermission" "accessClassInPackage.sun.nio.fs")
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.security.AccessController.checkPermission(AccessController.java:884)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkPackageAccess(SecurityManager.java:1564)
	at java.lang.Class.checkPackageAccess(Class.java:2372)
	at java.lang.Class.checkMemberAccess(Class.java:2351)
	at java.lang.Class.getDeclaredFields(Class.java:1915)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator$2.run(RamUsageEstimator.java:585)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator$2.run(RamUsageEstimator.java:582)
	at java.security.AccessController.doPrivileged(Native Method)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.createCacheEntry(RamUsageEstimator.java:582)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.measureSizeOf(RamUsageEstimator.java:545)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.sizeOfAll(RamUsageEstimator.java:387)
	at com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule$1.afterAlways(StaticFieldsInvariantRule.java:129)
	... 10 more


FAILED:  junit.framework.TestSuite.org.apache.solr.analytics.function.field.BooleanFieldsTest

Error Message:
Clean up static fields (in @AfterClass?) and null them, your test still has references to classes of which the sizes cannot be measured due to security restrictions or Java 9 module encapsulation:   - private static org.apache.solr.schema.IndexSchema org.apache.solr.analytics.function.field.AbstractAnalyticsFieldTest.indexSchema   - private static org.apache.solr.search.SolrIndexSearcher org.apache.solr.analytics.function.field.AbstractAnalyticsFieldTest.searcher

Stack Trace:
junit.framework.AssertionFailedError: Clean up static fields (in @AfterClass?) and null them, your test still has references to classes of which the sizes cannot be measured due to security restrictions or Java 9 module encapsulation:
  - private static org.apache.solr.schema.IndexSchema org.apache.solr.analytics.function.field.AbstractAnalyticsFieldTest.indexSchema
  - private static org.apache.solr.search.SolrIndexSearcher org.apache.solr.analytics.function.field.AbstractAnalyticsFieldTest.searcher
	at __randomizedtesting.SeedInfo.seed([2BB1AFE26D716F6F]:0)
	at com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule$1.afterAlways(StaticFieldsInvariantRule.java:146)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.security.AccessControlException: access denied ("java.lang.RuntimePermission" "accessClassInPackage.sun.nio.fs")
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.security.AccessController.checkPermission(AccessController.java:884)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkPackageAccess(SecurityManager.java:1564)
	at java.lang.Class.checkPackageAccess(Class.java:2372)
	at java.lang.Class.checkMemberAccess(Class.java:2351)
	at java.lang.Class.getDeclaredFields(Class.java:1915)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator$2.run(RamUsageEstimator.java:585)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator$2.run(RamUsageEstimator.java:582)
	at java.security.AccessController.doPrivileged(Native Method)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.createCacheEntry(RamUsageEstimator.java:582)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.measureSizeOf(RamUsageEstimator.java:545)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.sizeOfAll(RamUsageEstimator.java:387)
	at com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule$1.afterAlways(StaticFieldsInvariantRule.java:129)
	... 10 more


FAILED:  junit.framework.TestSuite.org.apache.solr.analytics.function.field.DateFieldsTest

Error Message:
Clean up static fields (in @AfterClass?) and null them, your test still has references to classes of which the sizes cannot be measured due to security restrictions or Java 9 module encapsulation:   - private static org.apache.solr.schema.IndexSchema org.apache.solr.analytics.function.field.AbstractAnalyticsFieldTest.indexSchema   - private static org.apache.solr.search.SolrIndexSearcher org.apache.solr.analytics.function.field.AbstractAnalyticsFieldTest.searcher

Stack Trace:
junit.framework.AssertionFailedError: Clean up static fields (in @AfterClass?) and null them, your test still has references to classes of which the sizes cannot be measured due to security restrictions or Java 9 module encapsulation:
  - private static org.apache.solr.schema.IndexSchema org.apache.solr.analytics.function.field.AbstractAnalyticsFieldTest.indexSchema
  - private static org.apache.solr.search.SolrIndexSearcher org.apache.solr.analytics.function.field.AbstractAnalyticsFieldTest.searcher
	at __randomizedtesting.SeedInfo.seed([2BB1AFE26D716F6F]:0)
	at com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule$1.afterAlways(StaticFieldsInvariantRule.java:146)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.security.AccessControlException: access denied ("java.lang.RuntimePermission" "accessClassInPackage.sun.nio.fs")
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.security.AccessController.checkPermission(AccessController.java:884)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkPackageAccess(SecurityManager.java:1564)
	at java.lang.Class.checkPackageAccess(Class.java:2372)
	at java.lang.Class.checkMemberAccess(Class.java:2351)
	at java.lang.Class.getDeclaredFields(Class.java:1915)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator$2.run(RamUsageEstimator.java:585)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator$2.run(RamUsageEstimator.java:582)
	at java.security.AccessController.doPrivileged(Native Method)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.createCacheEntry(RamUsageEstimator.java:582)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.measureSizeOf(RamUsageEstimator.java:545)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.sizeOfAll(RamUsageEstimator.java:387)
	at com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule$1.afterAlways(StaticFieldsInvariantRule.java:129)
	... 10 more


FAILED:  junit.framework.TestSuite.org.apache.solr.analytics.function.field.DoubleFieldsTest

Error Message:
Clean up static fields (in @AfterClass?) and null them, your test still has references to classes of which the sizes cannot be measured due to security restrictions or Java 9 module encapsulation:   - private static org.apache.solr.schema.IndexSchema org.apache.solr.analytics.function.field.AbstractAnalyticsFieldTest.indexSchema   - private static org.apache.solr.search.SolrIndexSearcher org.apache.solr.analytics.function.field.AbstractAnalyticsFieldTest.searcher

Stack Trace:
junit.framework.AssertionFailedError: Clean up static fields (in @AfterClass?) and null them, your test still has references to classes of which the sizes cannot be measured due to security restrictions or Java 9 module encapsulation:
  - private static org.apache.solr.schema.IndexSchema org.apache.solr.analytics.function.field.AbstractAnalyticsFieldTest.indexSchema
  - private static org.apache.solr.search.SolrIndexSearcher org.apache.solr.analytics.function.field.AbstractAnalyticsFieldTest.searcher
	at __randomizedtesting.SeedInfo.seed([2BB1AFE26D716F6F]:0)
	at com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule$1.afterAlways(StaticFieldsInvariantRule.java:146)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.security.AccessControlException: access denied ("java.lang.RuntimePermission" "accessClassInPackage.sun.nio.fs")
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.security.AccessController.checkPermission(AccessController.java:884)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkPackageAccess(SecurityManager.java:1564)
	at java.lang.Class.checkPackageAccess(Class.java:2372)
	at java.lang.Class.checkMemberAccess(Class.java:2351)
	at java.lang.Class.getDeclaredFields(Class.java:1915)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator$2.run(RamUsageEstimator.java:585)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator$2.run(RamUsageEstimator.java:582)
	at java.security.AccessController.doPrivileged(Native Method)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.createCacheEntry(RamUsageEstimator.java:582)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.measureSizeOf(RamUsageEstimator.java:545)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.sizeOfAll(RamUsageEstimator.java:387)
	at com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule$1.afterAlways(StaticFieldsInvariantRule.java:129)
	... 10 more


FAILED:  junit.framework.TestSuite.org.apache.solr.analytics.function.field.FloatFieldsTest

Error Message:
Clean up static fields (in @AfterClass?) and null them, your test still has references to classes of which the sizes cannot be measured due to security restrictions or Java 9 module encapsulation:   - private static org.apache.solr.schema.IndexSchema org.apache.solr.analytics.function.field.AbstractAnalyticsFieldTest.indexSchema   - private static org.apache.solr.search.SolrIndexSearcher org.apache.solr.analytics.function.field.AbstractAnalyticsFieldTest.searcher

Stack Trace:
junit.framework.AssertionFailedError: Clean up static fields (in @AfterClass?) and null them, your test still has references to classes of which the sizes cannot be measured due to security restrictions or Java 9 module encapsulation:
  - private static org.apache.solr.schema.IndexSchema org.apache.solr.analytics.function.field.AbstractAnalyticsFieldTest.indexSchema
  - private static org.apache.solr.search.SolrIndexSearcher org.apache.solr.analytics.function.field.AbstractAnalyticsFieldTest.searcher
	at __randomizedtesting.SeedInfo.seed([2BB1AFE26D716F6F]:0)
	at com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule$1.afterAlways(StaticFieldsInvariantRule.java:146)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.security.AccessControlException: access denied ("java.lang.RuntimePermission" "accessClassInPackage.sun.nio.fs")
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.security.AccessController.checkPermission(AccessController.java:884)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkPackageAccess(SecurityManager.java:1564)
	at java.lang.Class.checkPackageAccess(Class.java:2372)
	at java.lang.Class.checkMemberAccess(Class.java:2351)
	at java.lang.Class.getDeclaredFields(Class.java:1915)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator$2.run(RamUsageEstimator.java:585)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator$2.run(RamUsageEstimator.java:582)
	at java.security.AccessController.doPrivileged(Native Method)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.createCacheEntry(RamUsageEstimator.java:582)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.measureSizeOf(RamUsageEstimator.java:545)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.sizeOfAll(RamUsageEstimator.java:387)
	at com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule$1.afterAlways(StaticFieldsInvariantRule.java:129)
	... 10 more


FAILED:  junit.framework.TestSuite.org.apache.solr.analytics.function.field.IntFieldsTest

Error Message:
Clean up static fields (in @AfterClass?) and null them, your test still has references to classes of which the sizes cannot be measured due to security restrictions or Java 9 module encapsulation:   - private static org.apache.solr.schema.IndexSchema org.apache.solr.analytics.function.field.AbstractAnalyticsFieldTest.indexSchema   - private static org.apache.solr.search.SolrIndexSearcher org.apache.solr.analytics.function.field.AbstractAnalyticsFieldTest.searcher

Stack Trace:
junit.framework.AssertionFailedError: Clean up static fields (in @AfterClass?) and null them, your test still has references to classes of which the sizes cannot be measured due to security restrictions or Java 9 module encapsulation:
  - private static org.apache.solr.schema.IndexSchema org.apache.solr.analytics.function.field.AbstractAnalyticsFieldTest.indexSchema
  - private static org.apache.solr.search.SolrIndexSearcher org.apache.solr.analytics.function.field.AbstractAnalyticsFieldTest.searcher
	at __randomizedtesting.SeedInfo.seed([2BB1AFE26D716F6F]:0)
	at com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule$1.afterAlways(StaticFieldsInvariantRule.java:146)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.security.AccessControlException: access denied ("java.lang.RuntimePermission" "accessClassInPackage.sun.nio.fs")
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.security.AccessController.checkPermission(AccessController.java:884)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkPackageAccess(SecurityManager.java:1564)
	at java.lang.Class.checkPackageAccess(Class.java:2372)
	at java.lang.Class.checkMemberAccess(Class.java:2351)
	at java.lang.Class.getDeclaredFields(Class.java:1915)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator$2.run(RamUsageEstimator.java:585)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator$2.run(RamUsageEstimator.java:582)
	at java.security.AccessController.doPrivileged(Native Method)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.createCacheEntry(RamUsageEstimator.java:582)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.measureSizeOf(RamUsageEstimator.java:545)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.sizeOfAll(RamUsageEstimator.java:387)
	at com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule$1.afterAlways(StaticFieldsInvariantRule.java:129)
	... 10 more


FAILED:  junit.framework.TestSuite.org.apache.solr.analytics.function.field.LongFieldsTest

Error Message:
Clean up static fields (in @AfterClass?) and null them, your test still has references to classes of which the sizes cannot be measured due to security restrictions or Java 9 module encapsulation:   - private static org.apache.solr.schema.IndexSchema org.apache.solr.analytics.function.field.AbstractAnalyticsFieldTest.indexSchema   - private static org.apache.solr.search.SolrIndexSearcher org.apache.solr.analytics.function.field.AbstractAnalyticsFieldTest.searcher

Stack Trace:
junit.framework.AssertionFailedError: Clean up static fields (in @AfterClass?) and null them, your test still has references to classes of which the sizes cannot be measured due to security restrictions or Java 9 module encapsulation:
  - private static org.apache.solr.schema.IndexSchema org.apache.solr.analytics.function.field.AbstractAnalyticsFieldTest.indexSchema
  - private static org.apache.solr.search.SolrIndexSearcher org.apache.solr.analytics.function.field.AbstractAnalyticsFieldTest.searcher
	at __randomizedtesting.SeedInfo.seed([2BB1AFE26D716F6F]:0)
	at com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule$1.afterAlways(StaticFieldsInvariantRule.java:146)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.security.AccessControlException: access denied ("java.lang.RuntimePermission" "accessClassInPackage.sun.nio.fs")
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.security.AccessController.checkPermission(AccessController.java:884)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkPackageAccess(SecurityManager.java:1564)
	at java.lang.Class.checkPackageAccess(Class.java:2372)
	at java.lang.Class.checkMemberAccess(Class.java:2351)
	at java.lang.Class.getDeclaredFields(Class.java:1915)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator$2.run(RamUsageEstimator.java:585)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator$2.run(RamUsageEstimator.java:582)
	at java.security.AccessController.doPrivileged(Native Method)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.createCacheEntry(RamUsageEstimator.java:582)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.measureSizeOf(RamUsageEstimator.java:545)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.sizeOfAll(RamUsageEstimator.java:387)
	at com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule$1.afterAlways(StaticFieldsInvariantRule.java:129)
	... 10 more


FAILED:  junit.framework.TestSuite.org.apache.solr.analytics.function.field.StringFieldsTest

Error Message:
Clean up static fields (in @AfterClass?) and null them, your test still has references to classes of which the sizes cannot be measured due to security restrictions or Java 9 module encapsulation:   - private static org.apache.solr.schema.IndexSchema org.apache.solr.analytics.function.field.AbstractAnalyticsFieldTest.indexSchema   - private static org.apache.solr.search.SolrIndexSearcher org.apache.solr.analytics.function.field.AbstractAnalyticsFieldTest.searcher

Stack Trace:
junit.framework.AssertionFailedError: Clean up static fields (in @AfterClass?) and null them, your test still has references to classes of which the sizes cannot be measured due to security restrictions or Java 9 module encapsulation:
  - private static org.apache.solr.schema.IndexSchema org.apache.solr.analytics.function.field.AbstractAnalyticsFieldTest.indexSchema
  - private static org.apache.solr.search.SolrIndexSearcher org.apache.solr.analytics.function.field.AbstractAnalyticsFieldTest.searcher
	at __randomizedtesting.SeedInfo.seed([2BB1AFE26D716F6F]:0)
	at com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule$1.afterAlways(StaticFieldsInvariantRule.java:146)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.security.AccessControlException: access denied ("java.lang.RuntimePermission" "accessClassInPackage.sun.nio.fs")
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.security.AccessController.checkPermission(AccessController.java:884)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkPackageAccess(SecurityManager.java:1564)
	at java.lang.Class.checkPackageAccess(Class.java:2372)
	at java.lang.Class.checkMemberAccess(Class.java:2351)
	at java.lang.Class.getDeclaredFields(Class.java:1915)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator$2.run(RamUsageEstimator.java:585)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator$2.run(RamUsageEstimator.java:582)
	at java.security.AccessController.doPrivileged(Native Method)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.createCacheEntry(RamUsageEstimator.java:582)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.measureSizeOf(RamUsageEstimator.java:545)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.sizeOfAll(RamUsageEstimator.java:387)
	at com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule$1.afterAlways(StaticFieldsInvariantRule.java:129)
	... 10 more


FAILED:  junit.framework.TestSuite.org.apache.solr.prometheus.scraper.SolrStandaloneScraperTest

Error Message:
Clean up static fields (in @AfterClass?) and null them, your test still has references to classes of which the sizes cannot be measured due to security restrictions or Java 9 module encapsulation:   - private static org.apache.solr.prometheus.scraper.SolrStandaloneScraper org.apache.solr.prometheus.scraper.SolrStandaloneScraperTest.solrScraper   - private static org.apache.solr.client.solrj.impl.HttpSolrClient org.apache.solr.prometheus.scraper.SolrStandaloneScraperTest.solrClient

Stack Trace:
junit.framework.AssertionFailedError: Clean up static fields (in @AfterClass?) and null them, your test still has references to classes of which the sizes cannot be measured due to security restrictions or Java 9 module encapsulation:
  - private static org.apache.solr.prometheus.scraper.SolrStandaloneScraper org.apache.solr.prometheus.scraper.SolrStandaloneScraperTest.solrScraper
  - private static org.apache.solr.client.solrj.impl.HttpSolrClient org.apache.solr.prometheus.scraper.SolrStandaloneScraperTest.solrClient
	at __randomizedtesting.SeedInfo.seed([9998B031EB90E2A]:0)
	at com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule$1.afterAlways(StaticFieldsInvariantRule.java:146)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.security.AccessControlException: access denied ("java.lang.RuntimePermission" "accessClassInPackage.sun.net.www.protocol.jar")
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.security.AccessController.checkPermission(AccessController.java:884)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkPackageAccess(SecurityManager.java:1564)
	at java.lang.Class.checkPackageAccess(Class.java:2372)
	at java.lang.Class.checkMemberAccess(Class.java:2351)
	at java.lang.Class.getDeclaredFields(Class.java:1915)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator$2.run(RamUsageEstimator.java:585)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator$2.run(RamUsageEstimator.java:582)
	at java.security.AccessController.doPrivileged(Native Method)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.createCacheEntry(RamUsageEstimator.java:582)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.measureSizeOf(RamUsageEstimator.java:545)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.sizeOfAll(RamUsageEstimator.java:387)
	at com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule$1.afterAlways(StaticFieldsInvariantRule.java:129)
	... 10 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.MoveReplicaHDFSFailoverTest

Error Message:
failure to login: javax.security.auth.login.LoginException: Security Exception

Stack Trace:
org.apache.hadoop.security.KerberosAuthException: failure to login: javax.security.auth.login.LoginException: Security Exception
	at __randomizedtesting.SeedInfo.seed([6E82B774BA24A9AC]:0)
	at org.apache.hadoop.security.UserGroupInformation.doSubjectLogin(UserGroupInformation.java:1847)
	at org.apache.hadoop.security.UserGroupInformation.createLoginUser(UserGroupInformation.java:710)
	at org.apache.hadoop.security.UserGroupInformation.getLoginUser(UserGroupInformation.java:660)
	at org.apache.hadoop.security.UserGroupInformation.getCurrentUser(UserGroupInformation.java:571)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.<init>(FSNamesystem.java:767)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.<init>(FSNamesystem.java:733)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.format(NameNode.java:1140)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.format(NameNode.java:409)
	at org.apache.hadoop.hdfs.DFSTestUtil.formatNameNode(DFSTestUtil.java:261)
	at org.apache.hadoop.hdfs.MiniDFSCluster.configureNameService(MiniDFSCluster.java:1072)
	at org.apache.hadoop.hdfs.MiniDFSCluster.createNameNodesAndSetConf(MiniDFSCluster.java:958)
	at org.apache.hadoop.hdfs.MiniDFSCluster.initMiniDFSCluster(MiniDFSCluster.java:890)
	at org.apache.hadoop.hdfs.MiniDFSCluster.<init>(MiniDFSCluster.java:518)
	at org.apache.hadoop.hdfs.MiniDFSCluster$Builder.build(MiniDFSCluster.java:477)
	at org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:134)
	at org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:66)
	at org.apache.solr.cloud.MoveReplicaHDFSFailoverTest.setupClass(MoveReplicaHDFSFailoverTest.java:55)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:878)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: javax.security.auth.login.LoginException: Security Exception
	at javax.security.auth.login.LoginContext.invoke(LoginContext.java:840)
	at javax.security.auth.login.LoginContext.access$000(LoginContext.java:195)
	at javax.security.auth.login.LoginContext$4.run(LoginContext.java:682)
	at javax.security.auth.login.LoginContext$4.run(LoginContext.java:680)
	at java.security.AccessController.doPrivileged(Native Method)
	at javax.security.auth.login.LoginContext.invokePriv(LoginContext.java:680)
	at javax.security.auth.login.LoginContext.login(LoginContext.java:587)
	at org.apache.hadoop.security.UserGroupInformation$HadoopLoginContext.login(UserGroupInformation.java:1926)
	at org.apache.hadoop.security.UserGroupInformation.doSubjectLogin(UserGroupInformation.java:1837)
	... 40 more
Caused by: java.lang.SecurityException
	at javax.security.auth.login.LoginContext.invoke(LoginContext.java:841)
	... 48 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.OverseerCollectionConfigSetProcessorTest

Error Message:
Clean up static fields (in @AfterClass?) and null them, your test still has references to classes of which the sizes cannot be measured due to security restrictions or Java 9 module encapsulation:   - private static org.apache.solr.client.solrj.cloud.DistribStateManager org.apache.solr.cloud.OverseerCollectionConfigSetProcessorTest.stateManagerMock

Stack Trace:
junit.framework.AssertionFailedError: Clean up static fields (in @AfterClass?) and null them, your test still has references to classes of which the sizes cannot be measured due to security restrictions or Java 9 module encapsulation:
  - private static org.apache.solr.client.solrj.cloud.DistribStateManager org.apache.solr.cloud.OverseerCollectionConfigSetProcessorTest.stateManagerMock
	at __randomizedtesting.SeedInfo.seed([6E82B774BA24A9AC]:0)
	at com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule$1.afterAlways(StaticFieldsInvariantRule.java:146)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.security.AccessControlException: access denied ("java.lang.RuntimePermission" "accessClassInPackage.sun.reflect.generics.repository")
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.security.AccessController.checkPermission(AccessController.java:884)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkPackageAccess(SecurityManager.java:1564)
	at java.lang.Class.checkPackageAccess(Class.java:2372)
	at java.lang.Class.checkMemberAccess(Class.java:2351)
	at java.lang.Class.getDeclaredFields(Class.java:1915)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator$2.run(RamUsageEstimator.java:585)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator$2.run(RamUsageEstimator.java:582)
	at java.security.AccessController.doPrivileged(Native Method)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.createCacheEntry(RamUsageEstimator.java:582)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.measureSizeOf(RamUsageEstimator.java:545)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.sizeOfAll(RamUsageEstimator.java:387)
	at com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule$1.afterAlways(StaticFieldsInvariantRule.java:129)
	... 10 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.OverseerTest

Error Message:
Clean up static fields (in @AfterClass?) and null them, your test still has references to classes of which the sizes cannot be measured due to security restrictions or Java 9 module encapsulation:   - private static org.apache.solr.cloud.ZkTestServer org.apache.solr.cloud.OverseerTest.server

Stack Trace:
junit.framework.AssertionFailedError: Clean up static fields (in @AfterClass?) and null them, your test still has references to classes of which the sizes cannot be measured due to security restrictions or Java 9 module encapsulation:
  - private static org.apache.solr.cloud.ZkTestServer org.apache.solr.cloud.OverseerTest.server
	at __randomizedtesting.SeedInfo.seed([6E82B774BA24A9AC]:0)
	at com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule$1.afterAlways(StaticFieldsInvariantRule.java:146)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.security.AccessControlException: access denied ("java.lang.RuntimePermission" "accessClassInPackage.sun.nio.fs")
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.security.AccessController.checkPermission(AccessController.java:884)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkPackageAccess(SecurityManager.java:1564)
	at java.lang.Class.checkPackageAccess(Class.java:2372)
	at java.lang.Class.checkMemberAccess(Class.java:2351)
	at java.lang.Class.getDeclaredFields(Class.java:1915)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator$2.run(RamUsageEstimator.java:585)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator$2.run(RamUsageEstimator.java:582)
	at java.security.AccessController.doPrivileged(Native Method)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.createCacheEntry(RamUsageEstimator.java:582)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.measureSizeOf(RamUsageEstimator.java:545)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.sizeOfAll(RamUsageEstimator.java:387)
	at com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule$1.afterAlways(StaticFieldsInvariantRule.java:129)
	... 10 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.TestWithCollection

Error Message:
Clean up static fields (in @AfterClass?) and null them, your test still has references to classes of which the sizes cannot be measured due to security restrictions or Java 9 module encapsulation:   - private static org.apache.solr.client.solrj.cloud.SolrCloudManager org.apache.solr.cloud.TestWithCollection.cloudManager

Stack Trace:
junit.framework.AssertionFailedError: Clean up static fields (in @AfterClass?) and null them, your test still has references to classes of which the sizes cannot be measured due to security restrictions or Java 9 module encapsulation:
  - private static org.apache.solr.client.solrj.cloud.SolrCloudManager org.apache.solr.cloud.TestWithCollection.cloudManager
	at __randomizedtesting.SeedInfo.seed([6E82B774BA24A9AC]:0)
	at com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule$1.afterAlways(StaticFieldsInvariantRule.java:146)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.security.AccessControlException: access denied ("java.lang.RuntimePermission" "accessClassInPackage.sun.nio.fs")
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.security.AccessController.checkPermission(AccessController.java:884)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkPackageAccess(SecurityManager.java:1564)
	at java.lang.Class.checkPackageAccess(Class.java:2372)
	at java.lang.Class.checkMemberAccess(Class.java:2351)
	at java.lang.Class.getDeclaredFields(Class.java:1915)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator$2.run(RamUsageEstimator.java:585)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator$2.run(RamUsageEstimator.java:582)
	at java.security.AccessController.doPrivileged(Native Method)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.createCacheEntry(RamUsageEstimator.java:582)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.measureSizeOf(RamUsageEstimator.java:545)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.sizeOfAll(RamUsageEstimator.java:387)
	at com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule$1.afterAlways(StaticFieldsInvariantRule.java:129)
	... 10 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.api.collections.HdfsCollectionsAPIDistributedZkTest

Error Message:
failure to login: javax.security.auth.login.LoginException: Security Exception

Stack Trace:
org.apache.hadoop.security.KerberosAuthException: failure to login: javax.security.auth.login.LoginException: Security Exception
	at __randomizedtesting.SeedInfo.seed([6E82B774BA24A9AC]:0)
	at org.apache.hadoop.security.UserGroupInformation.doSubjectLogin(UserGroupInformation.java:1847)
	at org.apache.hadoop.security.UserGroupInformation.createLoginUser(UserGroupInformation.java:710)
	at org.apache.hadoop.security.UserGroupInformation.getLoginUser(UserGroupInformation.java:660)
	at org.apache.hadoop.security.UserGroupInformation.getCurrentUser(UserGroupInformation.java:571)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.<init>(FSNamesystem.java:767)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.<init>(FSNamesystem.java:733)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.format(NameNode.java:1140)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.format(NameNode.java:409)
	at org.apache.hadoop.hdfs.DFSTestUtil.formatNameNode(DFSTestUtil.java:261)
	at org.apache.hadoop.hdfs.MiniDFSCluster.configureNameService(MiniDFSCluster.java:1072)
	at org.apache.hadoop.hdfs.MiniDFSCluster.createNameNodesAndSetConf(MiniDFSCluster.java:958)
	at org.apache.hadoop.hdfs.MiniDFSCluster.initMiniDFSCluster(MiniDFSCluster.java:890)
	at org.apache.hadoop.hdfs.MiniDFSCluster.<init>(MiniDFSCluster.java:518)
	at org.apache.hadoop.hdfs.MiniDFSCluster$Builder.build(MiniDFSCluster.java:477)
	at org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:134)
	at org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:66)
	at org.apache.solr.cloud.api.collections.HdfsCollectionsAPIDistributedZkTest.setupClass(HdfsCollectionsAPIDistributedZkTest.java:41)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:878)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: javax.security.auth.login.LoginException: Security Exception
	at javax.security.auth.login.LoginContext.invoke(LoginContext.java:840)
	at javax.security.auth.login.LoginContext.access$000(LoginContext.java:195)
	at javax.security.auth.login.LoginContext$4.run(LoginContext.java:682)
	at javax.security.auth.login.LoginContext$4.run(LoginContext.java:680)
	at java.security.AccessController.doPrivileged(Native Method)
	at javax.security.auth.login.LoginContext.invokePriv(LoginContext.java:680)
	at javax.security.auth.login.LoginContext.login(LoginContext.java:587)
	at org.apache.hadoop.security.UserGroupInformation$HadoopLoginContext.login(UserGroupInformation.java:1926)
	at org.apache.hadoop.security.UserGroupInformation.doSubjectLogin(UserGroupInformation.java:1837)
	... 40 more
Caused by: java.lang.SecurityException
	at javax.security.auth.login.LoginContext.invoke(LoginContext.java:841)
	... 48 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.api.collections.TestHdfsCloudBackupRestore

Error Message:
failure to login: javax.security.auth.login.LoginException: Security Exception

Stack Trace:
org.apache.hadoop.security.KerberosAuthException: failure to login: javax.security.auth.login.LoginException: Security Exception
	at __randomizedtesting.SeedInfo.seed([6E82B774BA24A9AC]:0)
	at org.apache.hadoop.security.UserGroupInformation.doSubjectLogin(UserGroupInformation.java:1847)
	at org.apache.hadoop.security.UserGroupInformation.createLoginUser(UserGroupInformation.java:710)
	at org.apache.hadoop.security.UserGroupInformation.getLoginUser(UserGroupInformation.java:660)
	at org.apache.hadoop.security.UserGroupInformation.getCurrentUser(UserGroupInformation.java:571)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.<init>(FSNamesystem.java:767)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.<init>(FSNamesystem.java:733)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.format(NameNode.java:1140)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.format(NameNode.java:409)
	at org.apache.hadoop.hdfs.DFSTestUtil.formatNameNode(DFSTestUtil.java:261)
	at org.apache.hadoop.hdfs.MiniDFSCluster.configureNameService(MiniDFSCluster.java:1072)
	at org.apache.hadoop.hdfs.MiniDFSCluster.createNameNodesAndSetConf(MiniDFSCluster.java:958)
	at org.apache.hadoop.hdfs.MiniDFSCluster.initMiniDFSCluster(MiniDFSCluster.java:890)
	at org.apache.hadoop.hdfs.MiniDFSCluster.<init>(MiniDFSCluster.java:518)
	at org.apache.hadoop.hdfs.MiniDFSCluster$Builder.build(MiniDFSCluster.java:477)
	at org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:134)
	at org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:66)
	at org.apache.solr.cloud.api.collections.TestHdfsCloudBackupRestore.setupClass(TestHdfsCloudBackupRestore.java:110)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:878)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: javax.security.auth.login.LoginException: Security Exception
	at javax.security.auth.login.LoginContext.invoke(LoginContext.java:840)
	at javax.security.auth.login.LoginContext.access$000(LoginContext.java:195)
	at javax.security.auth.login.LoginContext$4.run(LoginContext.java:682)
	at javax.security.auth.login.LoginContext$4.run(LoginContext.java:680)
	at java.security.AccessController.doPrivileged(Native Method)
	at javax.security.auth.login.LoginContext.invokePriv(LoginContext.java:680)
	at javax.security.auth.login.LoginContext.login(LoginContext.java:587)
	at org.apache.hadoop.security.UserGroupInformation$HadoopLoginContext.login(UserGroupInformation.java:1926)
	at org.apache.hadoop.security.UserGroupInformation.doSubjectLogin(UserGroupInformation.java:1837)
	... 40 more
Caused by: java.lang.SecurityException
	at javax.security.auth.login.LoginContext.invoke(LoginContext.java:841)
	... 48 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.autoscaling.ComputePlanActionTest

Error Message:
Clean up static fields (in @AfterClass?) and null them, your test still has references to classes of which the sizes cannot be measured due to security restrictions or Java 9 module encapsulation:   - private static org.apache.solr.client.solrj.cloud.SolrCloudManager org.apache.solr.cloud.autoscaling.ComputePlanActionTest.cloudManager

Stack Trace:
junit.framework.AssertionFailedError: Clean up static fields (in @AfterClass?) and null them, your test still has references to classes of which the sizes cannot be measured due to security restrictions or Java 9 module encapsulation:
  - private static org.apache.solr.client.solrj.cloud.SolrCloudManager org.apache.solr.cloud.autoscaling.ComputePlanActionTest.cloudManager
	at __randomizedtesting.SeedInfo.seed([6E82B774BA24A9AC]:0)
	at com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule$1.afterAlways(StaticFieldsInvariantRule.java:146)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.security.AccessControlException: access denied ("java.lang.RuntimePermission" "accessClassInPackage.sun.nio.fs")
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.security.AccessController.checkPermission(AccessController.java:884)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkPackageAccess(SecurityManager.java:1564)
	at java.lang.Class.checkPackageAccess(Class.java:2372)
	at java.lang.Class.checkMemberAccess(Class.java:2351)
	at java.lang.Class.getDeclaredFields(Class.java:1915)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator$2.run(RamUsageEstimator.java:585)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator$2.run(RamUsageEstimator.java:582)
	at java.security.AccessController.doPrivileged(Native Method)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.createCacheEntry(RamUsageEstimator.java:582)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.measureSizeOf(RamUsageEstimator.java:545)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.sizeOfAll(RamUsageEstimator.java:387)
	at com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule$1.afterAlways(StaticFieldsInvariantRule.java:129)
	... 10 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.autoscaling.HdfsAutoAddReplicasIntegrationTest

Error Message:
failure to login: javax.security.auth.login.LoginException: Security Exception

Stack Trace:
org.apache.hadoop.security.KerberosAuthException: failure to login: javax.security.auth.login.LoginException: Security Exception
	at __randomizedtesting.SeedInfo.seed([6E82B774BA24A9AC]:0)
	at org.apache.hadoop.security.UserGroupInformation.doSubjectLogin(UserGroupInformation.java:1847)
	at org.apache.hadoop.security.UserGroupInformation.createLoginUser(UserGroupInformation.java:710)
	at org.apache.hadoop.security.UserGroupInformation.getLoginUser(UserGroupInformation.java:660)
	at org.apache.hadoop.security.UserGroupInformation.getCurrentUser(UserGroupInformation.java:571)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.<init>(FSNamesystem.java:767)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.<init>(FSNamesystem.java:733)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.format(NameNode.java:1140)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.format(NameNode.java:409)
	at org.apache.hadoop.hdfs.DFSTestUtil.formatNameNode(DFSTestUtil.java:261)
	at org.apache.hadoop.hdfs.MiniDFSCluster.configureNameService(MiniDFSCluster.java:1072)
	at org.apache.hadoop.hdfs.MiniDFSCluster.createNameNodesAndSetConf(MiniDFSCluster.java:958)
	at org.apache.hadoop.hdfs.MiniDFSCluster.initMiniDFSCluster(MiniDFSCluster.java:890)
	at org.apache.hadoop.hdfs.MiniDFSCluster.<init>(MiniDFSCluster.java:518)
	at org.apache.hadoop.hdfs.MiniDFSCluster$Builder.build(MiniDFSCluster.java:477)
	at org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:134)
	at org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:66)
	at org.apache.solr.cloud.autoscaling.HdfsAutoAddReplicasIntegrationTest.setupClass(HdfsAutoAddReplicasIntegrationTest.java:41)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:878)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: javax.security.auth.login.LoginException: Security Exception
	at javax.security.auth.login.LoginContext.invoke(LoginContext.java:840)
	at javax.security.auth.login.LoginContext.access$000(LoginContext.java:195)
	at javax.security.auth.login.LoginContext$4.run(LoginContext.java:682)
	at javax.security.auth.login.LoginContext$4.run(LoginContext.java:680)
	at java.security.AccessController.doPrivileged(Native Method)
	at javax.security.auth.login.LoginContext.invokePriv(LoginContext.java:680)
	at javax.security.auth.login.LoginContext.login(LoginContext.java:587)
	at org.apache.hadoop.security.UserGroupInformation$HadoopLoginContext.login(UserGroupInformation.java:1926)
	at org.apache.hadoop.security.UserGroupInformation.doSubjectLogin(UserGroupInformation.java:1837)
	... 40 more
Caused by: java.lang.SecurityException
	at javax.security.auth.login.LoginContext.invoke(LoginContext.java:841)
	... 48 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.autoscaling.IndexSizeTriggerTest

Error Message:
Clean up static fields (in @AfterClass?) and null them, your test still has references to classes of which the sizes cannot be measured due to security restrictions or Java 9 module encapsulation:   - private static org.apache.solr.core.SolrResourceLoader org.apache.solr.cloud.autoscaling.IndexSizeTriggerTest.loader

Stack Trace:
junit.framework.AssertionFailedError: Clean up static fields (in @AfterClass?) and null them, your test still has references to classes of which the sizes cannot be measured due to security restrictions or Java 9 module encapsulation:
  - private static org.apache.solr.core.SolrResourceLoader org.apache.solr.cloud.autoscaling.IndexSizeTriggerTest.loader
	at __randomizedtesting.SeedInfo.seed([6E82B774BA24A9AC]:0)
	at com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule$1.afterAlways(StaticFieldsInvariantRule.java:146)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.security.AccessControlException: access denied ("java.lang.RuntimePermission" "accessClassInPackage.sun.nio.fs")
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.security.AccessController.checkPermission(AccessController.java:884)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkPackageAccess(SecurityManager.java:1564)
	at java.lang.Class.checkPackageAccess(Class.java:2372)
	at java.lang.Class.checkMemberAccess(Class.java:2351)
	at java.lang.Class.getDeclaredFields(Class.java:1915)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator$2.run(RamUsageEstimator.java:585)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator$2.run(RamUsageEstimator.java:582)
	at java.security.AccessController.doPrivileged(Native Method)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.createCacheEntry(RamUsageEstimator.java:582)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.measureSizeOf(RamUsageEstimator.java:545)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.sizeOfAll(RamUsageEstimator.java:387)
	at com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule$1.afterAlways(StaticFieldsInvariantRule.java:129)
	... 10 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.autoscaling.NodeAddedTriggerIntegrationTest

Error Message:
Clean up static fields (in @AfterClass?) and null them, your test still has references to classes of which the sizes cannot be measured due to security restrictions or Java 9 module encapsulation:   - private static volatile org.apache.solr.client.solrj.cloud.SolrCloudManager org.apache.solr.cloud.autoscaling.NodeAddedTriggerIntegrationTest.cloudManager

Stack Trace:
junit.framework.AssertionFailedError: Clean up static fields (in @AfterClass?) and null them, your test still has references to classes of which the sizes cannot be measured due to security restrictions or Java 9 module encapsulation:
  - private static volatile org.apache.solr.client.solrj.cloud.SolrCloudManager org.apache.solr.cloud.autoscaling.NodeAddedTriggerIntegrationTest.cloudManager
	at __randomizedtesting.SeedInfo.seed([6E82B774BA24A9AC]:0)
	at com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule$1.afterAlways(StaticFieldsInvariantRule.java:146)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.security.AccessControlException: access denied ("java.lang.RuntimePermission" "accessClassInPackage.sun.nio.fs")
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.security.AccessController.checkPermission(AccessController.java:884)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkPackageAccess(SecurityManager.java:1564)
	at java.lang.Class.checkPackageAccess(Class.java:2372)
	at java.lang.Class.checkMemberAccess(Class.java:2351)
	at java.lang.Class.getDeclaredFields(Class.java:1915)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator$2.run(RamUsageEstimator.java:585)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator$2.run(RamUsageEstimator.java:582)
	at java.security.AccessController.doPrivileged(Native Method)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.createCacheEntry(RamUsageEstimator.java:582)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.measureSizeOf(RamUsageEstimator.java:545)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.sizeOfAll(RamUsageEstimator.java:387)
	at com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule$1.afterAlways(StaticFieldsInvariantRule.java:129)
	... 10 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.autoscaling.NodeLostTriggerIntegrationTest

Error Message:
Clean up static fields (in @AfterClass?) and null them, your test still has references to classes of which the sizes cannot be measured due to security restrictions or Java 9 module encapsulation:   - private static volatile org.apache.solr.client.solrj.cloud.SolrCloudManager org.apache.solr.cloud.autoscaling.NodeLostTriggerIntegrationTest.cloudManager

Stack Trace:
junit.framework.AssertionFailedError: Clean up static fields (in @AfterClass?) and null them, your test still has references to classes of which the sizes cannot be measured due to security restrictions or Java 9 module encapsulation:
  - private static volatile org.apache.solr.client.solrj.cloud.SolrCloudManager org.apache.solr.cloud.autoscaling.NodeLostTriggerIntegrationTest.cloudManager
	at __randomizedtesting.SeedInfo.seed([6E82B774BA24A9AC]:0)
	at com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule$1.afterAlways(StaticFieldsInvariantRule.java:146)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.security.AccessControlException: access denied ("java.lang.RuntimePermission" "accessClassInPackage.sun.nio.fs")
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.security.AccessController.checkPermission(AccessController.java:884)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkPackageAccess(SecurityManager.java:1564)
	at java.lang.Class.checkPackageAccess(Class.java:2372)
	at java.lang.Class.checkMemberAccess(Class.java:2351)
	at java.lang.Class.getDeclaredFields(Class.java:1915)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator$2.run(RamUsageEstimator.java:585)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator$2.run(RamUsageEstimator.java:582)
	at java.security.AccessController.doPrivileged(Native Method)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.createCacheEntry(RamUsageEstimator.java:582)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.measureSizeOf(RamUsageEstimator.java:545)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.sizeOfAll(RamUsageEstimator.java:387)
	at com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule$1.afterAlways(StaticFieldsInvariantRule.java:129)
	... 10 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.autoscaling.SearchRateTriggerIntegrationTest

Error Message:
Clean up static fields (in @AfterClass?) and null them, your test still has references to classes of which the sizes cannot be measured due to security restrictions or Java 9 module encapsulation:   - private static org.apache.solr.client.solrj.cloud.SolrCloudManager org.apache.solr.cloud.autoscaling.SearchRateTriggerIntegrationTest.cloudManager

Stack Trace:
junit.framework.AssertionFailedError: Clean up static fields (in @AfterClass?) and null them, your test still has references to classes of which the sizes cannot be measured due to security restrictions or Java 9 module encapsulation:
  - private static org.apache.solr.client.solrj.cloud.SolrCloudManager org.apache.solr.cloud.autoscaling.SearchRateTriggerIntegrationTest.cloudManager
	at __randomizedtesting.SeedInfo.seed([6E82B774BA24A9AC]:0)
	at com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule$1.afterAlways(StaticFieldsInvariantRule.java:146)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.security.AccessControlException: access denied ("java.lang.RuntimePermission" "accessClassInPackage.sun.nio.fs")
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.security.AccessController.checkPermission(AccessController.java:884)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkPackageAccess(SecurityManager.java:1564)
	at java.lang.Class.checkPackageAccess(Class.java:2372)
	at java.lang.Class.checkMemberAccess(Class.java:2351)
	at java.lang.Class.getDeclaredFields(Class.java:1915)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator$2.run(RamUsageEstimator.java:585)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator$2.run(RamUsageEstimator.java:582)
	at java.security.AccessController.doPrivileged(Native Method)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.createCacheEntry(RamUsageEstimator.java:582)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.measureSizeOf(RamUsageEstimator.java:545)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.sizeOfAll(RamUsageEstimator.java:387)
	at com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule$1.afterAlways(StaticFieldsInvariantRule.java:129)
	... 10 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.autoscaling.TriggerIntegrationTest

Error Message:
Clean up static fields (in @AfterClass?) and null them, your test still has references to classes of which the sizes cannot be measured due to security restrictions or Java 9 module encapsulation:   - private static org.apache.solr.client.solrj.cloud.SolrCloudManager org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.cloudManager

Stack Trace:
junit.framework.AssertionFailedError: Clean up static fields (in @AfterClass?) and null them, your test still has references to classes of which the sizes cannot be measured due to security restrictions or Java 9 module encapsulation:
  - private static org.apache.solr.client.solrj.cloud.SolrCloudManager org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.cloudManager
	at __randomizedtesting.SeedInfo.seed([6E82B774BA24A9AC]:0)
	at com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule$1.afterAlways(StaticFieldsInvariantRule.java:146)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.security.AccessControlException: access denied ("java.lang.RuntimePermission" "accessClassInPackage.sun.nio.fs")
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.security.AccessController.checkPermission(AccessController.java:884)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkPackageAccess(SecurityManager.java:1564)
	at java.lang.Class.checkPackageAccess(Class.java:2372)
	at java.lang.Class.checkMemberAccess(Class.java:2351)
	at java.lang.Class.getDeclaredFields(Class.java:1915)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator$2.run(RamUsageEstimator.java:585)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator$2.run(RamUsageEstimator.java:582)
	at java.security.AccessController.doPrivileged(Native Method)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.createCacheEntry(RamUsageEstimator.java:582)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.measureSizeOf(RamUsageEstimator.java:545)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.sizeOfAll(RamUsageEstimator.java:387)
	at com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule$1.afterAlways(StaticFieldsInvariantRule.java:129)
	... 10 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.autoscaling.sim.TestSimClusterStateProvider

Error Message:
Clean up static fields (in @AfterClass?) and null them, your test still has references to classes of which the sizes cannot be measured due to security restrictions or Java 9 module encapsulation:   - private static org.apache.solr.client.solrj.cloud.SolrCloudManager org.apache.solr.cloud.autoscaling.sim.TestSimClusterStateProvider.cloudManager

Stack Trace:
junit.framework.AssertionFailedError: Clean up static fields (in @AfterClass?) and null them, your test still has references to classes of which the sizes cannot be measured due to security restrictions or Java 9 module encapsulation:
  - private static org.apache.solr.client.solrj.cloud.SolrCloudManager org.apache.solr.cloud.autoscaling.sim.TestSimClusterStateProvider.cloudManager
	at __randomizedtesting.SeedInfo.seed([6E82B774BA24A9AC]:0)
	at com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule$1.afterAlways(StaticFieldsInvariantRule.java:146)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.security.AccessControlException: access denied ("java.lang.RuntimePermission" "accessClassInPackage.sun.nio.fs")
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.security.AccessController.checkPermission(AccessController.java:884)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkPackageAccess(SecurityManager.java:1564)
	at java.lang.Class.checkPackageAccess(Class.java:2372)
	at java.lang.Class.checkMemberAccess(Class.java:2351)
	at java.lang.Class.getDeclaredFields(Class.java:1915)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator$2.run(RamUsageEstimator.java:585)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator$2.run(RamUsageEstimator.java:582)
	at java.security.AccessController.doPrivileged(Native Method)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.createCacheEntry(RamUsageEstimator.java:582)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.measureSizeOf(RamUsageEstimator.java:545)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.sizeOfAll(RamUsageEstimator.java:387)
	at com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule$1.afterAlways(StaticFieldsInvariantRule.java:129)
	... 10 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.autoscaling.sim.TestSnapshotCloudManager

Error Message:
Clean up static fields (in @AfterClass?) and null them, your test still has references to classes of which the sizes cannot be measured due to security restrictions or Java 9 module encapsulation:   - private static org.apache.solr.client.solrj.cloud.SolrCloudManager org.apache.solr.cloud.autoscaling.sim.TestSnapshotCloudManager.realManager

Stack Trace:
junit.framework.AssertionFailedError: Clean up static fields (in @AfterClass?) and null them, your test still has references to classes of which the sizes cannot be measured due to security restrictions or Java 9 module encapsulation:
  - private static org.apache.solr.client.solrj.cloud.SolrCloudManager org.apache.solr.cloud.autoscaling.sim.TestSnapshotCloudManager.realManager
	at __randomizedtesting.SeedInfo.seed([6E82B774BA24A9AC]:0)
	at com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule$1.afterAlways(StaticFieldsInvariantRule.java:146)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.security.AccessControlException: access denied ("java.lang.RuntimePermission" "accessClassInPackage.sun.nio.fs")
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.security.AccessController.checkPermission(AccessController.java:884)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkPackageAccess(SecurityManager.java:1564)
	at java.lang.Class.checkPackageAccess(Class.java:2372)
	at java.lang.Class.checkMemberAccess(Class.java:2351)
	at java.lang.Class.getDeclaredFields(Class.java:1915)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator$2.run(RamUsageEstimator.java:585)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator$2.run(RamUsageEstimator.java:582)
	at java.security.AccessController.doPrivileged(Native Method)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.createCacheEntry(RamUsageEstimator.java:582)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.measureSizeOf(RamUsageEstimator.java:545)
	at com.carrotsearch.randomizedtesting.rules.RamUsageEstimator.sizeOfAll(RamUsageEstimator.java:387)
	at com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule$1.afterAlways(StaticFieldsInvariantRule.java:129)
	... 10 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HDFSCollectionsAPITest

Error Message:
failure to login: javax.security.auth.login.LoginException: Security Exception

Stack Trace:
org.apache.hadoop.security.KerberosAuthException: failure to login: javax.security.auth.login.LoginException: Security Exception
	at __randomizedtesting.SeedInfo.seed([6E82B774BA24A9AC]:0)
	at org.apache.hadoop.security.UserGroupInformation.doSubjectLogin(UserGroupInformation.java:1847)
	at org.apache.hadoop.security.UserGroupInformation.createLoginUser(UserGroupInformation.java:710)
	at org.apache.hadoop.security.UserGroupInformation.getLoginUser(UserGroupInformation.java:660)
	at org.apache.hadoop.security.UserGroupInformation.getCurrentUser(UserGroupInformation.java:571)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.<init>(FSNamesystem.java:767)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.<init>(FSNamesystem.java:733)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.format(NameNode.java:1140)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.format(NameNode.java:409)
	at org.apache.hadoop.hdfs.DFSTestUtil.formatNameNode(DFSTestUtil.java:261)
	at org.apache.hadoop.hdfs.MiniDFSCluster.configureNameService(MiniDFSCluster.java:1072)
	at org.apache.hadoop.hdfs.MiniDFSCluster.createNameNodesAndSetConf(MiniDFSCluster.java:958)
	at org.apache.hadoop.hdfs.MiniDFSCluster.initMiniDFSCluster(MiniDFSCluster.java:890)
	at org.apache.hadoop.hdfs.MiniDFSCluster.<init>(MiniDFSCluster.java:518)
	at org.apache.hadoop.hdfs.MiniDFSCluster$Builder.build(MiniDFSCluster.java:477)
	at org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:134)
	at org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:66)
	at org.apache.solr.cloud.hdfs.HDFSCollectionsAPITest.setupClass(HDFSCollectionsAPITest.java:46)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.random

[...truncated too long message...]

toryTest, LTFunctionTest, LongFieldsTest]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LongFieldsTest -Dtests.seed=2BB1AFE26D716F6F -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/test-data/enwiki.random.lines.txt -Dtests.locale=en-SG -Dtests.timezone=Pacific/Midway -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   0.00s J0 | LongFieldsTest (suite) <<<
   [junit4]    > Throwable #1: junit.framework.AssertionFailedError: Clean up static fields (in @AfterClass?) and null them, your test still has references to classes of which the sizes cannot be measured due to security restrictions or Java 9 module encapsulation:
   [junit4]    >   - private static org.apache.solr.schema.IndexSchema org.apache.solr.analytics.function.field.AbstractAnalyticsFieldTest.indexSchema
   [junit4]    >   - private static org.apache.solr.search.SolrIndexSearcher org.apache.solr.analytics.function.field.AbstractAnalyticsFieldTest.searcher
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([2BB1AFE26D716F6F]:0)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > Caused by: java.security.AccessControlException: access denied ("java.lang.RuntimePermission" "accessClassInPackage.sun.nio.fs")
   [junit4]    > 	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
   [junit4]    > 	at java.security.AccessController.checkPermission(AccessController.java:884)
   [junit4]    > 	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
   [junit4]    > 	at java.lang.SecurityManager.checkPackageAccess(SecurityManager.java:1564)
   [junit4]    > 	at java.lang.Class.checkPackageAccess(Class.java:2372)
   [junit4]    > 	at java.lang.Class.checkMemberAccess(Class.java:2351)
   [junit4]    > 	at java.lang.Class.getDeclaredFields(Class.java:1915)
   [junit4]    > 	at java.security.AccessController.doPrivileged(Native Method)
   [junit4]    > 	... 10 more
   [junit4] Completed [39/78 (7!)] on J0 in 0.51s, 5 tests, 1 failure <<< FAILURES!

[...truncated 4 lines...]
   [junit4] Suite: org.apache.solr.analytics.function.field.StringFieldsTest
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene84): {double_dm_t=PostingsFormat(name=LuceneVarGapFixedInterval), long_l_t=PostingsFormat(name=MockRandom), float_fm_t=Lucene84, double_d_t=Lucene84, long_lm_t=FST50, date_dtm_t=FST50, int_im_t=FST50, boolean_bm=PostingsFormat(name=MockRandom), float_f_t=PostingsFormat(name=LuceneVarGapFixedInterval), date_dt_t=PostingsFormat(name=MockRandom), string_s=FST50, boolean_b=Lucene84, id=Lucene84, int_i_t=PostingsFormat(name=LuceneVarGapFixedInterval), string_sm=PostingsFormat(name=LuceneVarGapFixedInterval)}, docValues:{long_l_t=DocValuesFormat(name=Asserting), int_im_p=DocValuesFormat(name=Lucene80), double_d_t=DocValuesFormat(name=Lucene80), long_l_p=DocValuesFormat(name=Asserting), long_lm_t=DocValuesFormat(name=Lucene80), double_d_p=DocValuesFormat(name=Lucene80), long_lm_p=DocValuesFormat(name=Lucene80), date_dtm_t=DocValuesFormat(name=Lucene80), int_im_t=DocValuesFormat(name=Lucene80), date_dt_p=DocValuesFormat(name=Asserting), float_f_t=DocValuesFormat(name=Direct), date_dt_t=DocValuesFormat(name=Asserting), float_f_p=DocValuesFormat(name=Direct), id=DocValuesFormat(name=Lucene80), double_dm_t=DocValuesFormat(name=Direct), float_fm_t=DocValuesFormat(name=Lucene80), double_dm_p=DocValuesFormat(name=Direct), float_fm_p=DocValuesFormat(name=Lucene80), boolean_bm=DocValuesFormat(name=Asserting), date_dtm_p=DocValuesFormat(name=Lucene80), string_s=DocValuesFormat(name=Lucene80), int_i_p=DocValuesFormat(name=Direct), boolean_b=DocValuesFormat(name=Lucene80), int_i_t=DocValuesFormat(name=Direct), string_sm=DocValuesFormat(name=Direct)}, maxPointsInLeafNode=335, maxMBSortInHeap=6.427237471205723, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@149b9bd5), locale=sr-Latn-ME, timezone=Asia/Dhaka
   [junit4]   2> NOTE: Linux 4.15.0-54-generic amd64/Oracle Corporation 1.8.0_191 (64-bit)/cpus=4,threads=1,free=228022584,total=397410304
   [junit4]   2> NOTE: All tests run in this JVM: [CastingDateValueStreamTest, RangeFacetTest, FillMissingFunctionTest, PowerFunctionTest, GTEFunctionTest, CastingIntValueStreamTest, BooleanFieldsTest, LegacyFacetSortingTest, RoundFunctionTest, LegacyQueryFacetTest, FloatFieldsTest, OverallAnalyticsTest, MultFunctionTest, IntFieldsTest, StringFieldsTest]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=StringFieldsTest -Dtests.seed=2BB1AFE26D716F6F -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/test-data/enwiki.random.lines.txt -Dtests.locale=sr-Latn-ME -Dtests.timezone=Asia/Dhaka -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   0.00s J2 | StringFieldsTest (suite) <<<
   [junit4]    > Throwable #1: junit.framework.AssertionFailedError: Clean up static fields (in @AfterClass?) and null them, your test still has references to classes of which the sizes cannot be measured due to security restrictions or Java 9 module encapsulation:
   [junit4]    >   - private static org.apache.solr.schema.IndexSchema org.apache.solr.analytics.function.field.AbstractAnalyticsFieldTest.indexSchema
   [junit4]    >   - private static org.apache.solr.search.SolrIndexSearcher org.apache.solr.analytics.function.field.AbstractAnalyticsFieldTest.searcher
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([2BB1AFE26D716F6F]:0)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > Caused by: java.security.AccessControlException: access denied ("java.lang.RuntimePermission" "accessClassInPackage.sun.nio.fs")
   [junit4]    > 	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
   [junit4]    > 	at java.security.AccessController.checkPermission(AccessController.java:884)
   [junit4]    > 	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
   [junit4]    > 	at java.lang.SecurityManager.checkPackageAccess(SecurityManager.java:1564)
   [junit4]    > 	at java.lang.Class.checkPackageAccess(Class.java:2372)
   [junit4]    > 	at java.lang.Class.checkMemberAccess(Class.java:2351)
   [junit4]    > 	at java.lang.Class.getDeclaredFields(Class.java:1915)
   [junit4]    > 	at java.security.AccessController.doPrivileged(Native Method)
   [junit4]    > 	... 10 more
   [junit4] Completed [41/78 (8!)] on J2 in 0.76s, 3 tests, 1 failure <<< FAILURES!

[...truncated 5668 lines...]
   [junit4] Suite: org.apache.solr.prometheus.scraper.SolrStandaloneScraperTest
   [junit4]   1> 13:47:47.261 [SUITE-SolrStandaloneScraperTest-seed#[9998B031EB90E2A]-worker] ERROR org.apache.solr.util.StartupLoggingUtils - Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene84): {}, docValues:{}, maxPointsInLeafNode=1508, maxMBSortInHeap=7.725262861565197, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@483b93aa), locale=be, timezone=America/Jujuy
   [junit4]   2> NOTE: Linux 4.15.0-54-generic amd64/Oracle Corporation 1.8.0_191 (64-bit)/cpus=4,threads=1,free=217534200,total=282591232
   [junit4]   2> NOTE: All tests run in this JVM: [SolrStandaloneScraperTest]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=SolrStandaloneScraperTest -Dtests.seed=9998B031EB90E2A -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/test-data/enwiki.random.lines.txt -Dtests.locale=be -Dtests.timezone=America/Jujuy -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   0.00s J2 | SolrStandaloneScraperTest (suite) <<<
   [junit4]    > Throwable #1: junit.framework.AssertionFailedError: Clean up static fields (in @AfterClass?) and null them, your test still has references to classes of which the sizes cannot be measured due to security restrictions or Java 9 module encapsulation:
   [junit4]    >   - private static org.apache.solr.prometheus.scraper.SolrStandaloneScraper org.apache.solr.prometheus.scraper.SolrStandaloneScraperTest.solrScraper
   [junit4]    >   - private static org.apache.solr.client.solrj.impl.HttpSolrClient org.apache.solr.prometheus.scraper.SolrStandaloneScraperTest.solrClient
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([9998B031EB90E2A]:0)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > Caused by: java.security.AccessControlException: access denied ("java.lang.RuntimePermission" "accessClassInPackage.sun.net.www.protocol.jar")
   [junit4]    > 	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
   [junit4]    > 	at java.security.AccessController.checkPermission(AccessController.java:884)
   [junit4]    > 	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
   [junit4]    > 	at java.lang.SecurityManager.checkPackageAccess(SecurityManager.java:1564)
   [junit4]    > 	at java.lang.Class.checkPackageAccess(Class.java:2372)
   [junit4]    > 	at java.lang.Class.checkMemberAccess(Class.java:2351)
   [junit4]    > 	at java.lang.Class.getDeclaredFields(Class.java:1915)
   [junit4]    > 	at java.security.AccessController.doPrivileged(Native Method)
   [junit4]    > 	... 10 more
   [junit4] Completed [4/5 (1!)] on J2 in 15.26s, 5 tests, 1 failure <<< FAILURES!

[...truncated 43971 lines...]