You are viewing a plain text version of this content. The canonical link for it is here.
Posted to builds@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2019/12/12 08:03:30 UTC

[JENKINS-EA] Lucene-Solr-master-Linux (64bit/jdk-14-ea+24) - Build # 25146 - Unstable!

Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Linux/25146/
Java: 64bit/jdk-14-ea+24 -XX:-UseCompressedOops -XX:+UseG1GC

1 tests failed.
FAILED:  org.apache.solr.cloud.DeleteShardTest.test

Error Message:
Error from server at https://127.0.0.1:36547/solr: The slice: shard2 is currently active. Only non-active (or custom-hashed) slices can be deleted.

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:36547/solr: The slice: shard2 is currently active. Only non-active (or custom-hashed) slices can be deleted.
	at __randomizedtesting.SeedInfo.seed([565BFE4F53AC2C56:DE0FC195FD5041AE]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:665)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:265)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:368)
	at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:296)
	at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.sendRequest(BaseCloudSolrClient.java:1143)
	at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.requestWithRetryOnStaleState(BaseCloudSolrClient.java:906)
	at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.request(BaseCloudSolrClient.java:838)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:207)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:224)
	at org.apache.solr.cloud.DeleteShardTest.test(DeleteShardTest.java:85)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.base/java.lang.Thread.run(Thread.java:833)




Build Log:
[...truncated 14040 lines...]
   [junit4] Suite: org.apache.solr.cloud.DeleteShardTest
   [junit4]   2> 1067205 INFO  (SUITE-DeleteShardTest-seed#[565BFE4F53AC2C56]-worker) [     ] o.a.s.SolrTestCaseJ4 Created dataDir: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DeleteShardTest_565BFE4F53AC2C56-001/data-dir-128-001
   [junit4]   2> 1067206 INFO  (SUITE-DeleteShardTest-seed#[565BFE4F53AC2C56]-worker) [     ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 1067206 INFO  (SUITE-DeleteShardTest-seed#[565BFE4F53AC2C56]-worker) [     ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason="", ssl=0.0/0.0, value=0.0/0.0, clientAuth=0.0/0.0)
   [junit4]   2> 1067207 INFO  (SUITE-DeleteShardTest-seed#[565BFE4F53AC2C56]-worker) [     ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 1067210 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.SolrTestCaseJ4 ###Starting test
   [junit4]   2> 1067211 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DeleteShardTest_565BFE4F53AC2C56-001/tempDir-001
   [junit4]   2> 1067211 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1067211 INFO  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1067211 INFO  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1067311 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.ZkTestServer start zk server on port:37469
   [junit4]   2> 1067311 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.ZkTestServer waitForServerUp: 127.0.0.1:37469
   [junit4]   2> 1067312 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:37469
   [junit4]   2> 1067312 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 37469
   [junit4]   2> 1067317 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1067341 INFO  (zkConnectionManagerCallback-9872-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1067341 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1067385 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1067398 INFO  (zkConnectionManagerCallback-9874-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1067398 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1067405 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1067412 INFO  (zkConnectionManagerCallback-9876-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1067412 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1067528 WARN  (jetty-launcher-9877-thread-1) [     ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 1067528 INFO  (jetty-launcher-9877-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 1067528 INFO  (jetty-launcher-9877-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 1067528 INFO  (jetty-launcher-9877-thread-1) [     ] o.e.j.s.Server jetty-9.4.19.v20190610; built: 2019-06-10T16:30:51.723Z; git: afcf563148970e98786327af5e07c261fda175d3; jvm 14-ea+24-1107
   [junit4]   2> 1067544 WARN  (jetty-launcher-9877-thread-2) [     ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 1067544 INFO  (jetty-launcher-9877-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 1067544 INFO  (jetty-launcher-9877-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 1067544 INFO  (jetty-launcher-9877-thread-2) [     ] o.e.j.s.Server jetty-9.4.19.v20190610; built: 2019-06-10T16:30:51.723Z; git: afcf563148970e98786327af5e07c261fda175d3; jvm 14-ea+24-1107
   [junit4]   2> 1067606 INFO  (jetty-launcher-9877-thread-1) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1067606 INFO  (jetty-launcher-9877-thread-1) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1067606 INFO  (jetty-launcher-9877-thread-1) [     ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 1067614 INFO  (jetty-launcher-9877-thread-1) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3800d98c{/solr,null,AVAILABLE}
   [junit4]   2> 1067615 INFO  (jetty-launcher-9877-thread-1) [     ] o.e.j.s.AbstractConnector Started ServerConnector@348d3a88{ssl,[ssl, alpn, http/1.1, h2]}{127.0.0.1:36547}
   [junit4]   2> 1067615 INFO  (jetty-launcher-9877-thread-1) [     ] o.e.j.s.Server Started @1067650ms
   [junit4]   2> 1067615 INFO  (jetty-launcher-9877-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=36547}
   [junit4]   2> 1067615 ERROR (jetty-launcher-9877-thread-1) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1067615 INFO  (jetty-launcher-9877-thread-1) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 1067615 INFO  (jetty-launcher-9877-thread-1) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 9.0.0
   [junit4]   2> 1067615 INFO  (jetty-launcher-9877-thread-1) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1067615 INFO  (jetty-launcher-9877-thread-1) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1067616 INFO  (jetty-launcher-9877-thread-1) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-12-12T07:26:00.712005Z
   [junit4]   2> 1067618 INFO  (jetty-launcher-9877-thread-2) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1067618 INFO  (jetty-launcher-9877-thread-2) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1067618 INFO  (jetty-launcher-9877-thread-2) [     ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 1067626 INFO  (jetty-launcher-9877-thread-2) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@21671b0f{/solr,null,AVAILABLE}
   [junit4]   2> 1067626 INFO  (jetty-launcher-9877-thread-1) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1067627 INFO  (jetty-launcher-9877-thread-2) [     ] o.e.j.s.AbstractConnector Started ServerConnector@25f90ff0{ssl,[ssl, alpn, http/1.1, h2]}{127.0.0.1:33457}
   [junit4]   2> 1067627 INFO  (jetty-launcher-9877-thread-2) [     ] o.e.j.s.Server Started @1067662ms
   [junit4]   2> 1067627 INFO  (jetty-launcher-9877-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=33457}
   [junit4]   2> 1067627 ERROR (jetty-launcher-9877-thread-2) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1067627 INFO  (jetty-launcher-9877-thread-2) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 1067627 INFO  (jetty-launcher-9877-thread-2) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 9.0.0
   [junit4]   2> 1067627 INFO  (jetty-launcher-9877-thread-2) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1067627 INFO  (jetty-launcher-9877-thread-2) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1067627 INFO  (jetty-launcher-9877-thread-2) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-12-12T07:26:00.723439Z
   [junit4]   2> 1067630 INFO  (zkConnectionManagerCallback-9879-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1067630 INFO  (jetty-launcher-9877-thread-1) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1067630 INFO  (jetty-launcher-9877-thread-1) [     ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1067653 INFO  (jetty-launcher-9877-thread-2) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1067671 INFO  (zkConnectionManagerCallback-9881-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1067671 INFO  (jetty-launcher-9877-thread-2) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1067672 INFO  (jetty-launcher-9877-thread-2) [     ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1068012 INFO  (jetty-launcher-9877-thread-2) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
   [junit4]   2> 1068013 WARN  (jetty-launcher-9877-thread-2) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@5dcccdb4[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1068013 WARN  (jetty-launcher-9877-thread-2) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@5dcccdb4[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1068021 WARN  (jetty-launcher-9877-thread-2) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@7f256270[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1068021 WARN  (jetty-launcher-9877-thread-2) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@7f256270[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1068022 INFO  (jetty-launcher-9877-thread-2) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:37469/solr
   [junit4]   2> 1068024 INFO  (jetty-launcher-9877-thread-2) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1068024 INFO  (zkConnectionManagerCallback-9889-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1068024 INFO  (jetty-launcher-9877-thread-2) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1068112 INFO  (jetty-launcher-9877-thread-1) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
   [junit4]   2> 1068113 WARN  (jetty-launcher-9877-thread-1) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@742042e0[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1068113 WARN  (jetty-launcher-9877-thread-1) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@742042e0[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1068122 WARN  (jetty-launcher-9877-thread-1) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@1862647a[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1068122 WARN  (jetty-launcher-9877-thread-1) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@1862647a[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1068123 INFO  (jetty-launcher-9877-thread-1) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:37469/solr
   [junit4]   2> 1068152 INFO  (jetty-launcher-9877-thread-1) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1068152 INFO  (jetty-launcher-9877-thread-2) [n:127.0.0.1:33457_solr     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1068154 INFO  (zkConnectionManagerCallback-9895-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1068154 INFO  (jetty-launcher-9877-thread-1) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1068162 INFO  (zkConnectionManagerCallback-9897-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1068162 INFO  (jetty-launcher-9877-thread-2) [n:127.0.0.1:33457_solr     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1068214 INFO  (jetty-launcher-9877-thread-2) [n:127.0.0.1:33457_solr     ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:33457_solr
   [junit4]   2> 1068215 INFO  (jetty-launcher-9877-thread-2) [n:127.0.0.1:33457_solr     ] o.a.s.c.Overseer Overseer (id=72169736756330503-127.0.0.1:33457_solr-n_0000000000) starting
   [junit4]   2> 1068217 INFO  (OverseerStateUpdate-72169736756330503-127.0.0.1:33457_solr-n_0000000000) [n:127.0.0.1:33457_solr     ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:33457_solr
   [junit4]   2> 1068218 INFO  (jetty-launcher-9877-thread-2) [n:127.0.0.1:33457_solr     ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33457_solr
   [junit4]   2> 1068219 WARN  (jetty-launcher-9877-thread-2) [n:127.0.0.1:33457_solr     ] o.a.s.c.CoreContainer Not all security plugins configured!  authentication=disabled authorization=disabled.  Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external.  See https://s.apache.org/solrsecurity for more info
   [junit4]   2> 1068222 INFO  (zkCallback-9896-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1068234 INFO  (jetty-launcher-9877-thread-2) [n:127.0.0.1:33457_solr     ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 1068249 INFO  (jetty-launcher-9877-thread-2) [n:127.0.0.1:33457_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33457.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@61231b3f
   [junit4]   2> 1068255 INFO  (jetty-launcher-9877-thread-2) [n:127.0.0.1:33457_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33457.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@61231b3f
   [junit4]   2> 1068255 INFO  (jetty-launcher-9877-thread-2) [n:127.0.0.1:33457_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33457.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@61231b3f
   [junit4]   2> 1068256 INFO  (jetty-launcher-9877-thread-1) [n:127.0.0.1:36547_solr     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1068256 INFO  (jetty-launcher-9877-thread-2) [n:127.0.0.1:33457_solr     ] o.a.s.p.PackageLoader /packages.json updated to version -1
   [junit4]   2> 1068256 INFO  (zkConnectionManagerCallback-9903-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1068256 INFO  (jetty-launcher-9877-thread-1) [n:127.0.0.1:36547_solr     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1068257 INFO  (jetty-launcher-9877-thread-2) [n:127.0.0.1:33457_solr     ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DeleteShardTest_565BFE4F53AC2C56-001/tempDir-001/node2/.
   [junit4]   2> 1068259 INFO  (jetty-launcher-9877-thread-1) [n:127.0.0.1:36547_solr     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1068261 INFO  (jetty-launcher-9877-thread-1) [n:127.0.0.1:36547_solr     ] o.a.s.c.ZkController Publish node=127.0.0.1:36547_solr as DOWN
   [junit4]   2> 1068262 INFO  (jetty-launcher-9877-thread-1) [n:127.0.0.1:36547_solr     ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1068262 INFO  (jetty-launcher-9877-thread-1) [n:127.0.0.1:36547_solr     ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:36547_solr
   [junit4]   2> 1068262 INFO  (zkCallback-9896-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1068262 INFO  (zkCallback-9902-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1068263 WARN  (jetty-launcher-9877-thread-1) [n:127.0.0.1:36547_solr     ] o.a.s.c.CoreContainer Not all security plugins configured!  authentication=disabled authorization=disabled.  Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external.  See https://s.apache.org/solrsecurity for more info
   [junit4]   2> 1068269 INFO  (jetty-launcher-9877-thread-1) [n:127.0.0.1:36547_solr     ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 1068284 INFO  (jetty-launcher-9877-thread-1) [n:127.0.0.1:36547_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_36547.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@61231b3f
   [junit4]   2> 1068294 INFO  (jetty-launcher-9877-thread-1) [n:127.0.0.1:36547_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_36547.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@61231b3f
   [junit4]   2> 1068294 INFO  (jetty-launcher-9877-thread-1) [n:127.0.0.1:36547_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_36547.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@61231b3f
   [junit4]   2> 1068300 INFO  (jetty-launcher-9877-thread-1) [n:127.0.0.1:36547_solr     ] o.a.s.p.PackageLoader /packages.json updated to version -1
   [junit4]   2> 1068301 INFO  (jetty-launcher-9877-thread-1) [n:127.0.0.1:36547_solr     ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DeleteShardTest_565BFE4F53AC2C56-001/tempDir-001/node1/.
   [junit4]   2> 1068318 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.MiniSolrCloudCluster waitForAllNodes: numServers=2
   [junit4]   2> 1068319 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1068320 INFO  (zkConnectionManagerCallback-9912-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1068320 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1068322 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1068323 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:37469/solr ready
   [junit4]   2> 1068339 INFO  (qtp2071450029-22167) [n:127.0.0.1:36547_solr     ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf&name=deleteShard&nrtReplicas=1&action=CREATE&numShards=2&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1068341 INFO  (OverseerThreadFactory-6026-thread-1-processing-n:127.0.0.1:33457_solr) [n:127.0.0.1:33457_solr     ] o.a.s.c.a.c.CreateCollectionCmd Create collection deleteShard
   [junit4]   2> 1068446 INFO  (OverseerStateUpdate-72169736756330503-127.0.0.1:33457_solr-n_0000000000) [n:127.0.0.1:33457_solr     ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"deleteShard",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"deleteShard_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:33457/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 1068447 INFO  (OverseerStateUpdate-72169736756330503-127.0.0.1:33457_solr-n_0000000000) [n:127.0.0.1:33457_solr     ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"deleteShard",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"deleteShard_shard2_replica_n2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:36547/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 1068658 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr    x:deleteShard_shard2_replica_n2 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=deleteShard_shard2_replica_n2&action=CREATE&numShards=2&collection=deleteShard&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1068661 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr    x:deleteShard_shard1_replica_n1 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=deleteShard_shard1_replica_n1&action=CREATE&numShards=2&collection=deleteShard&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1068662 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr    x:deleteShard_shard1_replica_n1 ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1069668 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.0.0
   [junit4]   2> 1069670 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.0.0
   [junit4]   2> 1069674 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.s.IndexSchema [deleteShard_shard2_replica_n2] Schema name=minimal
   [junit4]   2> 1069675 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1069675 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.c.CoreContainer Creating SolrCore 'deleteShard_shard2_replica_n2' using configuration from collection deleteShard, trusted=true
   [junit4]   2> 1069676 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_36547.solr.core.deleteShard.shard2.replica_n2' (registry 'solr.core.deleteShard.shard2.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@61231b3f
   [junit4]   2> 1069677 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.s.IndexSchema [deleteShard_shard1_replica_n1] Schema name=minimal
   [junit4]   2> 1069677 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.c.SolrCore [[deleteShard_shard2_replica_n2] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DeleteShardTest_565BFE4F53AC2C56-001/tempDir-001/node1/deleteShard_shard2_replica_n2], dataDir=[/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DeleteShardTest_565BFE4F53AC2C56-001/tempDir-001/node1/./deleteShard_shard2_replica_n2/data/]
   [junit4]   2> 1069678 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1069678 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.c.CoreContainer Creating SolrCore 'deleteShard_shard1_replica_n1' using configuration from collection deleteShard, trusted=true
   [junit4]   2> 1069679 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33457.solr.core.deleteShard.shard1.replica_n1' (registry 'solr.core.deleteShard.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@61231b3f
   [junit4]   2> 1069679 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.c.SolrCore [[deleteShard_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DeleteShardTest_565BFE4F53AC2C56-001/tempDir-001/node2/deleteShard_shard1_replica_n1], dataDir=[/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DeleteShardTest_565BFE4F53AC2C56-001/tempDir-001/node2/./deleteShard_shard1_replica_n1/data/]
   [junit4]   2> 1069705 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1069705 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1069705 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1069705 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1069707 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1069707 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1069707 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1069707 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1069707 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.s.SolrIndexSearcher Opening [Searcher@63cd4fe2[deleteShard_shard1_replica_n1] main]
   [junit4]   2> 1069707 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.s.SolrIndexSearcher Opening [Searcher@1654af20[deleteShard_shard2_replica_n2] main]
   [junit4]   2> 1069708 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 1069708 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 1069708 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 1069708 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 1069709 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 1069709 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1652697923903815680
   [junit4]   2> 1069709 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 1069709 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1652697923903815680
   [junit4]   2> 1069711 INFO  (searcherExecutor-6035-thread-1-processing-n:127.0.0.1:36547_solr x:deleteShard_shard2_replica_n2 c:deleteShard s:shard2 r:core_node4) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.c.SolrCore [deleteShard_shard2_replica_n2] Registered new searcher Searcher@1654af20[deleteShard_shard2_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1069711 INFO  (searcherExecutor-6036-thread-1-processing-n:127.0.0.1:33457_solr x:deleteShard_shard1_replica_n1 c:deleteShard s:shard1 r:core_node3) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.c.SolrCore [deleteShard_shard1_replica_n1] Registered new searcher Searcher@63cd4fe2[deleteShard_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1069713 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.c.ZkShardTerms Successful update of terms at /collections/deleteShard/terms/shard2 to Terms{values={core_node4=0}, version=0}
   [junit4]   2> 1069713 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/deleteShard/leaders/shard2
   [junit4]   2> 1069714 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.c.ZkShardTerms Successful update of terms at /collections/deleteShard/terms/shard1 to Terms{values={core_node3=0}, version=0}
   [junit4]   2> 1069714 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/deleteShard/leaders/shard1
   [junit4]   2> 1069715 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1069715 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1069715 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:36547/solr/deleteShard_shard2_replica_n2/
   [junit4]   2> 1069718 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 1069718 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.c.SyncStrategy https://127.0.0.1:36547/solr/deleteShard_shard2_replica_n2/ has no replicas
   [junit4]   2> 1069718 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/deleteShard/leaders/shard2/leader after winning as /collections/deleteShard/leader_elect/shard2/election/72169736756330504-core_node4-n_0000000000
   [junit4]   2> 1069719 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:36547/solr/deleteShard_shard2_replica_n2/ shard2
   [junit4]   2> 1069721 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1069721 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1069721 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:33457/solr/deleteShard_shard1_replica_n1/
   [junit4]   2> 1069721 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 1069721 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.c.SyncStrategy https://127.0.0.1:33457/solr/deleteShard_shard1_replica_n1/ has no replicas
   [junit4]   2> 1069721 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/deleteShard/leaders/shard1/leader after winning as /collections/deleteShard/leader_elect/shard1/election/72169736756330503-core_node3-n_0000000000
   [junit4]   2> 1069722 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:33457/solr/deleteShard_shard1_replica_n1/ shard1
   [junit4]   2> 1069823 INFO  (zkCallback-9902-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1069823 INFO  (zkCallback-9896-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1069824 INFO  (zkCallback-9902-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1069824 INFO  (zkCallback-9896-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1069824 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1069825 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1069827 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=deleteShard_shard2_replica_n2&action=CREATE&numShards=2&collection=deleteShard&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1169
   [junit4]   2> 1069827 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=deleteShard_shard1_replica_n1&action=CREATE&numShards=2&collection=deleteShard&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1165
   [junit4]   2> 1069828 INFO  (qtp2071450029-22167) [n:127.0.0.1:36547_solr     ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
   [junit4]   2> 1069927 INFO  (zkCallback-9902-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1069927 INFO  (zkCallback-9902-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1069927 INFO  (zkCallback-9896-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1069927 INFO  (zkCallback-9896-thread-3) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1069927 INFO  (zkCallback-9902-thread-3) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1069928 INFO  (zkCallback-9896-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1069928 INFO  (qtp2071450029-22167) [n:127.0.0.1:36547_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf&name=deleteShard&nrtReplicas=1&action=CREATE&numShards=2&wt=javabin&version=2} status=0 QTime=1589
   [junit4]   2> 1069928 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.MiniSolrCloudCluster waitForActiveCollection: deleteShard
   [junit4]   2> 1069935 INFO  (qtp2071450029-22165) [n:127.0.0.1:36547_solr     ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :deleteshard with params action=DELETESHARD&collection=deleteShard&shard=shard1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1069937 INFO  (OverseerThreadFactory-6026-thread-2-processing-n:127.0.0.1:33457_solr) [n:127.0.0.1:33457_solr c:deleteShard s:shard1   ] o.a.s.c.a.c.DeleteShardCmd Delete shard invoked
   [junit4]   2> 1069938 ERROR (OverseerThreadFactory-6026-thread-2-processing-n:127.0.0.1:33457_solr) [n:127.0.0.1:33457_solr c:deleteShard s:shard1   ] o.a.s.c.a.c.OverseerCollectionMessageHandler Collection: deleteShard operation: deleteshard failed:org.apache.solr.common.SolrException: The slice: shard1 is currently active. Only non-active (or custom-hashed) slices can be deleted.
   [junit4]   2> 	at org.apache.solr.cloud.api.collections.DeleteShardCmd.call(DeleteShardCmd.java:87)
   [junit4]   2> 	at org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler.processMessage(OverseerCollectionMessageHandler.java:263)
   [junit4]   2> 	at org.apache.solr.cloud.OverseerTaskProcessor$Runner.run(OverseerTaskProcessor.java:505)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
   [junit4]   2> 	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
   [junit4]   2> 	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
   [junit4]   2> 	at java.base/java.lang.Thread.run(Thread.java:833)
   [junit4]   2> 
   [junit4]   2> 1069938 INFO  (OverseerCollectionConfigSetProcessor-72169736756330503-127.0.0.1:33457_solr-n_0000000000) [n:127.0.0.1:33457_solr     ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1069941 INFO  (qtp2071450029-22165) [n:127.0.0.1:36547_solr c:deleteShard    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=DELETESHARD&collection=deleteShard&shard=shard1&wt=javabin&version=2} status=400 QTime=6
   [junit4]   2> 1069943 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.SolrCloudTestCase waitForState (deleteShard): Expected shard shard1 to be in state inactive
   [junit4]   2> 1069943 INFO  (OverseerStateUpdate-72169736756330503-127.0.0.1:33457_solr-n_0000000000) [n:127.0.0.1:33457_solr     ] o.a.s.c.o.SliceMutator Update shard state invoked for collection: deleteShard with message: {
   [junit4]   2>   "shard1":"inactive",
   [junit4]   2>   "collection":"deleteShard",
   [junit4]   2>   "operation":"updateshardstate"}
   [junit4]   2> 1069943 INFO  (OverseerStateUpdate-72169736756330503-127.0.0.1:33457_solr-n_0000000000) [n:127.0.0.1:33457_solr     ] o.a.s.c.o.SliceMutator Update shard state shard1 to inactive
   [junit4]   2> 1070044 INFO  (zkCallback-9896-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1070044 INFO  (zkCallback-9911-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1070044 INFO  (zkCallback-9896-thread-3) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1070044 INFO  (zkCallback-9896-thread-4) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1070044 INFO  (zkCallback-9902-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1070044 INFO  (zkCallback-9902-thread-3) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1070044 INFO  (zkCallback-9902-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1070044 INFO  (zkCallback-9911-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1070045 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr     ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :deleteshard with params action=DELETESHARD&collection=deleteShard&shard=shard1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1070046 INFO  (OverseerThreadFactory-6026-thread-3-processing-n:127.0.0.1:33457_solr) [n:127.0.0.1:33457_solr c:deleteShard s:shard1   ] o.a.s.c.a.c.DeleteShardCmd Delete shard invoked
   [junit4]   2> 1070046 INFO  (OverseerThreadFactory-6026-thread-3-processing-n:127.0.0.1:33457_solr) [n:127.0.0.1:33457_solr c:deleteShard s:shard1   ] o.a.s.c.a.c.DeleteShardCmd Deleting replica for collection=deleteShard shard=shard1 on node=127.0.0.1:33457_solr
   [junit4]   2> 1070046 INFO  (OverseerCollectionConfigSetProcessor-72169736756330503-127.0.0.1:33457_solr-n_0000000000) [n:127.0.0.1:33457_solr     ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1070047 INFO  (qtp1778644731-22166) [n:127.0.0.1:33457_solr    x:deleteShard_shard1_replica_n1 ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.deleteShard.shard1.replica_n1, tag=null
   [junit4]   2> 1070047 INFO  (qtp1778644731-22166) [n:127.0.0.1:33457_solr    x:deleteShard_shard1_replica_n1 ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@67edf51: rootName = solr_33457, domain = solr.core.deleteShard.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.deleteShard.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@48925ba4
   [junit4]   2> 1070060 INFO  (qtp1778644731-22166) [n:127.0.0.1:33457_solr    x:deleteShard_shard1_replica_n1 ] o.a.s.c.SolrCore [deleteShard_shard1_replica_n1]  CLOSING SolrCore org.apache.solr.core.SolrCore@1106cd65
   [junit4]   2> 1070060 INFO  (qtp1778644731-22166) [n:127.0.0.1:33457_solr    x:deleteShard_shard1_replica_n1 ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.deleteShard.shard1.replica_n1, tag=SolrCore@1106cd65
   [junit4]   2> 1070060 INFO  (qtp1778644731-22166) [n:127.0.0.1:33457_solr    x:deleteShard_shard1_replica_n1 ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.deleteShard.shard1.leader, tag=SolrCore@1106cd65
   [junit4]   2> 1070061 INFO  (qtp1778644731-22166) [n:127.0.0.1:33457_solr    x:deleteShard_shard1_replica_n1 ] o.a.s.c.ZkShardTerms Successful update of terms at /collections/deleteShard/terms/shard1 to Terms{values={}, version=1}
   [junit4]   2> 1070062 INFO  (qtp1778644731-22166) [n:127.0.0.1:33457_solr    x:deleteShard_shard1_replica_n1 ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={deleteInstanceDir=true&core=deleteShard_shard1_replica_n1&deleteMetricsHistory=true&qt=/admin/cores&deleteDataDir=true&action=UNLOAD&wt=javabin&version=2&deleteIndex=true} status=0 QTime=15
   [junit4]   2> 1070163 INFO  (zkCallback-9902-thread-3) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1070163 INFO  (zkCallback-9902-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1070163 INFO  (zkCallback-9902-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1070163 INFO  (zkCallback-9896-thread-4) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1070163 INFO  (zkCallback-9896-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1070163 INFO  (zkCallback-9896-thread-3) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1070164 INFO  (OverseerStateUpdate-72169736756330503-127.0.0.1:33457_solr-n_0000000000) [n:127.0.0.1:33457_solr     ] o.a.s.c.o.CollectionMutator Removing collection: deleteShard shard: shard1 from clusterstate
   [junit4]   2> 1070265 INFO  (zkCallback-9896-thread-3) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1070265 INFO  (zkCallback-9896-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1070265 INFO  (zkCallback-9896-thread-4) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1070265 INFO  (zkCallback-9896-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1070265 INFO  (zkCallback-9902-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1070265 INFO  (zkCallback-9902-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1070265 INFO  (zkCallback-9902-thread-3) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1070266 INFO  (OverseerThreadFactory-6026-thread-3-processing-n:127.0.0.1:33457_solr) [n:127.0.0.1:33457_solr c:deleteShard s:shard1   ] o.a.s.c.a.c.DeleteShardCmd Successfully deleted collection: deleteShard, shard: shard1
   [junit4]   2> 1070266 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=DELETESHARD&collection=deleteShard&shard=shard1&wt=javabin&version=2} status=0 QTime=221
   [junit4]   2> 1070267 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.SolrCloudTestCase waitForState (deleteShard): Expected 'shard1' to be removed
   [junit4]   2> 1070268 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.SolrCloudTestCase waitForState (deleteShard): Expected shard shard2 to be in state construction
   [junit4]   2> 1070268 INFO  (OverseerStateUpdate-72169736756330503-127.0.0.1:33457_solr-n_0000000000) [n:127.0.0.1:33457_solr     ] o.a.s.c.o.SliceMutator Update shard state invoked for collection: deleteShard with message: {
   [junit4]   2>   "shard2":"construction",
   [junit4]   2>   "collection":"deleteShard",
   [junit4]   2>   "operation":"updateshardstate"}
   [junit4]   2> 1070268 INFO  (OverseerStateUpdate-72169736756330503-127.0.0.1:33457_solr-n_0000000000) [n:127.0.0.1:33457_solr     ] o.a.s.c.o.SliceMutator Update shard state shard2 to construction
   [junit4]   2> 1070369 INFO  (zkCallback-9902-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1070369 INFO  (zkCallback-9911-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1070369 INFO  (zkCallback-9902-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1070369 INFO  (zkCallback-9902-thread-3) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1070369 INFO  (zkCallback-9911-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1070370 INFO  (qtp2071450029-22167) [n:127.0.0.1:36547_solr     ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :deleteshard with params action=DELETESHARD&collection=deleteShard&shard=shard2&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1070371 INFO  (OverseerThreadFactory-6026-thread-4-processing-n:127.0.0.1:33457_solr) [n:127.0.0.1:33457_solr c:deleteShard s:shard2   ] o.a.s.c.a.c.DeleteShardCmd Delete shard invoked
   [junit4]   2> 1070371 ERROR (OverseerThreadFactory-6026-thread-4-processing-n:127.0.0.1:33457_solr) [n:127.0.0.1:33457_solr c:deleteShard s:shard2   ] o.a.s.c.a.c.OverseerCollectionMessageHandler Collection: deleteShard operation: deleteshard failed:org.apache.solr.common.SolrException: The slice: shard2 is currently active. Only non-active (or custom-hashed) slices can be deleted.
   [junit4]   2> 	at org.apache.solr.cloud.api.collections.DeleteShardCmd.call(DeleteShardCmd.java:87)
   [junit4]   2> 	at org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler.processMessage(OverseerCollectionMessageHandler.java:263)
   [junit4]   2> 	at org.apache.solr.cloud.OverseerTaskProcessor$Runner.run(OverseerTaskProcessor.java:505)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
   [junit4]   2> 	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
   [junit4]   2> 	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
   [junit4]   2> 	at java.base/java.lang.Thread.run(Thread.java:833)
   [junit4]   2> 
   [junit4]   2> 1070371 INFO  (OverseerCollectionConfigSetProcessor-72169736756330503-127.0.0.1:33457_solr-n_0000000000) [n:127.0.0.1:33457_solr     ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1070373 INFO  (qtp2071450029-22167) [n:127.0.0.1:36547_solr c:deleteShard    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=DELETESHARD&collection=deleteShard&shard=shard2&wt=javabin&version=2} status=400 QTime=3
   [junit4]   2> 1070475 INFO  (jetty-closer-9915-thread-1) [     ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1320324481
   [junit4]   2> 1070475 INFO  (jetty-closer-9915-thread-1) [     ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:33457_solr
   [junit4]   2> 1070476 INFO  (jetty-closer-9915-thread-1) [     ] o.a.s.c.ZkController Publish this node as DOWN...
   [junit4]   2> 1070476 INFO  (jetty-closer-9915-thread-1) [     ] o.a.s.c.ZkController Publish node=127.0.0.1:33457_solr as DOWN
   [junit4]   2> 1070476 INFO  (jetty-closer-9915-thread-2) [     ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=134538154
   [junit4]   2> 1070476 INFO  (jetty-closer-9915-thread-2) [     ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:36547_solr
   [junit4]   2> 1070476 INFO  (jetty-closer-9915-thread-2) [     ] o.a.s.c.ZkController Publish this node as DOWN...
   [junit4]   2> 1070476 INFO  (jetty-closer-9915-thread-2) [     ] o.a.s.c.ZkController Publish node=127.0.0.1:36547_solr as DOWN
   [junit4]   2> 1070477 INFO  (jetty-closer-9915-thread-1) [     ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 1070477 INFO  (jetty-closer-9915-thread-1) [     ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@4446e372: rootName = solr_33457, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@81cf508
   [junit4]   2> 1070481 INFO  (jetty-closer-9915-thread-1) [     ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 1070481 INFO  (jetty-closer-9915-thread-1) [     ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@fef8ab6: rootName = solr_33457, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@39d8c401
   [junit4]   2> 1070482 INFO  (coreCloseExecutor-6046-thread-1) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.c.SolrCore [deleteShard_shard2_replica_n2]  CLOSING SolrCore org.apache.solr.core.SolrCore@77976d35
   [junit4]   2> 1070482 INFO  (coreCloseExecutor-6046-thread-1) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.deleteShard.shard2.replica_n2, tag=SolrCore@77976d35
   [junit4]   2> 1070482 INFO  (coreCloseExecutor-6046-thread-1) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@359201bd: rootName = solr_36547, domain = solr.core.deleteShard.shard2.replica_n2, service url = null, agent id = null] for registry solr.core.deleteShard.shard2.replica_n2 / com.codahale.metrics.MetricRegistry@594f48a7
   [junit4]   2> 1070482 INFO  (jetty-closer-9915-thread-1) [     ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 1070482 INFO  (jetty-closer-9915-thread-1) [     ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@2a570f0b: rootName = solr_33457, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@5195fe2
   [junit4]   2> 1070483 INFO  (jetty-closer-9915-thread-1) [     ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 1070483 INFO  (closeThreadPool-9922-thread-2) [     ] o.a.s.c.Overseer Overseer (id=72169736756330503-127.0.0.1:33457_solr-n_0000000000) closing
   [junit4]   2> 1070483 INFO  (OverseerStateUpdate-72169736756330503-127.0.0.1:33457_solr-n_0000000000) [n:127.0.0.1:33457_solr     ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:33457_solr
   [junit4]   2> 1070483 INFO  (OverseerAutoScalingTriggerThread-72169736756330503-127.0.0.1:33457_solr-n_0000000000) [     ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 1070486 INFO  (closeThreadPool-9922-thread-1) [     ] o.a.s.c.Overseer Overseer (id=72169736756330503-127.0.0.1:33457_solr-n_0000000000) closing
   [junit4]   2> 1070486 INFO  (zkCallback-9902-thread-3) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
   [junit4]   2> 1070490 INFO  (coreCloseExecutor-6046-thread-1) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.deleteShard.shard2.leader, tag=SolrCore@77976d35
   [junit4]   2> 1070491 INFO  (jetty-closer-9915-thread-2) [     ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 1070491 INFO  (jetty-closer-9915-thread-2) [     ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@29cd5ae1: rootName = solr_36547, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@562f42d6
   [junit4]   2> 1070495 INFO  (jetty-closer-9915-thread-2) [     ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 1070495 INFO  (jetty-closer-9915-thread-2) [     ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@53d4764b: rootName = solr_36547, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@39d8c401
   [junit4]   2> 1070497 INFO  (jetty-closer-9915-thread-2) [     ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 1070497 INFO  (jetty-closer-9915-thread-2) [     ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@146125e0: rootName = solr_36547, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@5195fe2
   [junit4]   2> 1070497 INFO  (jetty-closer-9915-thread-2) [     ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 1070586 INFO  (jetty-closer-9915-thread-1) [     ] o.a.s.c.Overseer Overseer (id=72169736756330503-127.0.0.1:33457_solr-n_0000000000) closing
   [junit4]   2> 1070587 INFO  (jetty-closer-9915-thread-1) [     ] o.e.j.s.AbstractConnector Stopped ServerConnector@25f90ff0{ssl,[ssl, alpn, http/1.1, h2]}{127.0.0.1:0}
   [junit4]   2> 1070587 INFO  (jetty-closer-9915-thread-1) [     ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@21671b0f{/solr,null,UNAVAILABLE}
   [junit4]   2> 1070587 INFO  (jetty-closer-9915-thread-1) [     ] o.e.j.s.session node0 Stopped scavenging
   [junit4]   2> 1070687 INFO  (jetty-closer-9915-thread-2) [     ] o.e.j.s.AbstractConnector Stopped ServerConnector@348d3a88{ssl,[ssl, alpn, http/1.1, h2]}{127.0.0.1:0}
   [junit4]   2> 1070687 INFO  (jetty-closer-9915-thread-2) [     ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@3800d98c{/solr,null,UNAVAILABLE}
   [junit4]   2> 1070688 INFO  (jetty-closer-9915-thread-2) [     ] o.e.j.s.session node0 Stopped scavenging
   [junit4]   2> 1070688 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.ZkTestServer Shutting down ZkTestServer.
   [junit4]   2> 1070890 WARN  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	3	/solr/collections/deleteShard/terms/shard1
   [junit4]   2> 	3	/solr/aliases.json
   [junit4]   2> 	2	/solr/packages.json
   [junit4]   2> 	2	/solr/security.json
   [junit4]   2> 	2	/solr/configs/conf
   [junit4]   2> 	2	/solr/collections/deleteShard/terms/shard2
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	49	/solr/collections/deleteShard/state.json
   [junit4]   2> 	3	/solr/clusterstate.json
   [junit4]   2> 	3	/solr/clusterprops.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	25	/solr/overseer/queue
   [junit4]   2> 	15	/solr/overseer/collection-queue-work
   [junit4]   2> 	7	/solr/live_nodes
   [junit4]   2> 	6	/solr/collections
   [junit4]   2> 
   [junit4]   2> 1070891 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.ZkTestServer waitForServerDown: 127.0.0.1:37469
   [junit4]   2> 1070891 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:37469
   [junit4]   2> 1070891 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 37469
   [junit4]   2> 1070892 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.SolrTestCaseJ4 ###Ending test
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=DeleteShardTest -Dtests.method=test -Dtests.seed=565BFE4F53AC2C56 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=ee -Dtests.timezone=Singapore -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   3.69s J1 | DeleteShardTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:36547/solr: The slice: shard2 is currently active. Only non-active (or custom-hashed) slices can be deleted.
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([565BFE4F53AC2C56:DE0FC195FD5041AE]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:665)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:265)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:368)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:296)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.sendRequest(BaseCloudSolrClient.java:1143)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.requestWithRetryOnStaleState(BaseCloudSolrClient.java:906)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.request(BaseCloudSolrClient.java:838)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:207)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:224)
   [junit4]    > 	at org.apache.solr.cloud.DeleteShardTest.test(DeleteShardTest.java:85)
   [junit4]    > 	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]    > 	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]    > 	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]    > 	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
   [junit4]    > 	at java.base/java.lang.Thread.run(Thread.java:833)
   [junit4]   2> 1070895 INFO  (TEST-DeleteShardTest.testDirectoryCleanupAfterDeleteShard-seed#[565BFE4F53AC2C56]) [     ] o.a.s.SolrTestCaseJ4 ###Starting testDirectoryCleanupAfterDeleteShard
   [junit4]   2> 1070895 INFO  (TEST-DeleteShardTest.testDirectoryCleanupAfterDeleteShard-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DeleteShardTest_565BFE4F53AC2C56-001/tempDir-002
   [junit4]   2> 1070896 INFO  (TEST-DeleteShardTest.testDirectoryCleanupAfterDeleteShard-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1070896 INFO  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1070896 INFO  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1070996 INFO  (TEST-DeleteShardTest.testDirectoryCleanupAfterDeleteShard-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.ZkTestServer start zk server on port:45323
   [junit4]   2> 1070996 INFO  (TEST-DeleteShardTest.testDirectoryCleanupAfterDeleteShard-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.ZkTestServer waitForServerUp: 127.0.0.1:45323
   [junit4]   2> 1070996 INFO  (TEST-DeleteShardTest.testDirectoryCleanupAfterDeleteShard-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:45323
   [junit4]   2> 1070996 INFO  (TEST-DeleteShardTest.testDirectoryCleanupAfterDeleteShard-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 45323
   [junit4]   2> 1070997 INFO  (TEST-DeleteShardTest.testDirectoryCleanupAfterDeleteShard-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1070998 INFO  (zkConnectionManagerCallback-9925-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1070998 INFO  (TEST-DeleteShardTest.testDirectoryCleanupAfterDeleteShard-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1071000 INFO  (TEST-DeleteShardTest.testDirectoryCleanupAfterDeleteShard-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1071001 INFO  (zkConnectionManagerCallback-9927-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1071001 INFO  (TEST-DeleteShardTest.testDirectoryCleanupAfterDeleteShard-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1071001 INFO  (TEST-DeleteShardTest.testDirectoryCleanupAfterDeleteShard-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1071002 INFO  (zkConnectionManagerCallback-9929-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1071002 INFO  (TEST-DeleteShardTest.testDirectoryCleanupAfterDeleteShard-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1071105 WARN  (jetty-launcher-9930-thread-1) [     ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 1071106 INFO  (jetty-launcher-9930-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 1071106 INFO  (jetty-launcher-9930-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 1071106 INFO  (jetty-launcher-9930-thread-1) [     ] o.e.j.s.Server jetty-9.4.19.v20190610; built: 2019-06-10T16:30:51.723Z; git: afcf563148970e98786327af5e07c261fda175d3; jvm 14-ea+24-1107
   [junit4]   2> 1071106 WARN  (jetty-launcher-9930-thread-2) [     ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 1071106 INFO  (jetty-launcher-9930-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 1071106 INFO  (jetty-launcher-9930-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 1071106 INFO  (jetty-launcher-9930-thread-2) [     ] o.e.j.s.Server jetty-9.4.19.v20190610; built: 2019-06-10T16:30:51.723Z; git: afcf563148970e98786327af5e07c261fda175d3; jvm 14-ea+24-1107
   [junit4]   2> 1071108 INFO  (jetty-launcher-9930-thread-1) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1071108 INFO  (jetty-launcher-9930-thread-1) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1071108 INFO  (jetty-launcher-9930-thread-1) [     ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 1071110 INFO  (jetty-launcher-9930-thread-1) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6b7d72a0{/solr,null,AVAILABLE}
   [junit4]   2> 1071115 INFO  (jetty-launcher-9930-thread-1) [     ] o.e.j.s.AbstractConnector Started ServerConnector@4189c1a7{ssl,[ssl, alpn, http/1.1, h2]}{127.0.0.1:45739}
   [junit4]   2> 1071115 INFO  (jetty-launcher-9930-thread-1) [     ] o.e.j.s.Server Started @1071150ms
   [junit4]   2> 1071115 INFO  (jetty-launcher-9930-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=45739}
   [junit4]   2> 1071116 ERROR (jetty-launcher-9930-thread-1) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1071116 INFO  (jetty-launcher-9930-thread-1) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 1071116 INFO  (jetty-launcher-9930-thread-1) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 9.0.0
   [junit4]   2> 1071116 INFO  (jetty-launcher-9930-thread-1) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1071116 INFO  (jetty-launcher-9930-thread-1) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1071116 INFO  (jetty-launcher-9930-thread-1) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-12-12T07:26:04.212145Z
   [junit4]   2> 1071124 INFO  (jetty-launcher-9930-thread-1) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1071124 INFO  (jetty-launcher-9930-thread-2) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1071124 INFO  (jetty-launcher-9930-thread-2) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1071124 INFO  (jetty-launcher-9930-thread-2) [     ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 1071125 INFO  (zkConnectionManagerCallback-9932-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1071125 INFO  (jetty-launcher-9930-thread-2) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4cf92585{/solr,null,AVAILABLE}
   [junit4]   2> 1071125 INFO  (jetty-launcher-9930-thread-1) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1071126 INFO  (jetty-launcher-9930-thread-1) [     ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1071126 INFO  (jetty-launcher-9930-thread-2) [     ] o.e.j.s.AbstractConnector Started ServerConnector@201fc22e{ssl,[ssl, alpn, http/1.1, h2]}{127.0.0.1:37321}
   [junit4]   2> 1071126 INFO  (jetty-launcher-9930-thread-2) [     ] o.e.j.s.Server Started @1071161ms
   [junit4]   2> 1071126 INFO  (jetty-launcher-9930-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=37321}
   [junit4]   2> 1071127 ERROR (jetty-launcher-9930-thread-2) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1071127 INFO  (jetty-launcher-9930-thread-2) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 1071127 INFO  (jetty-launcher-9930-thread-2) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 9.0.0
   [junit4]   2> 1071127 INFO  (jetty-launcher-9930-thread-2) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1071127 INFO  (jetty-launcher-9930-thread-2) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1071127 INFO  (jetty-launcher-9930-thread-2) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-12-12T07:26:04.223096Z
   [junit4]   2> 1071130 INFO  (jetty-launcher-9930-thread-2) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1071131 INFO  (zkConnectionManagerCallback-9934-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1071131 INFO  (jetty-launcher-9930-thread-2) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1071131 INFO  (jetty-launcher-9930-thread-2) [     ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1071326 INFO  (jetty-launcher-9930-thread-1) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
   [junit4]   2> 1071327 WARN  (jetty-launcher-9930-thread-1) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@45e5d0a5[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1071327 WARN  (jetty-launcher-9930-thread-1) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@45e5d0a5[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1071336 WARN  (jetty-launcher-9930-thread-1) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@4656cc40[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1071336 WARN  (jetty-launcher-9930-thread-1) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@4656cc40[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1071337 INFO  (jetty-launcher-9930-thread-1) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45323/solr
   [junit4]   2> 1071338 INFO  (jetty-launcher-9930-thread-1) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1071341 INFO  (zkConnectionManagerCallback-9941-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1071342 INFO  (jetty-launcher-9930-thread-1) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1071386 INFO  (jetty-launcher-9930-thread-2) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
   [junit4]   2> 1071387 WARN  (jetty-launcher-9930-thread-2) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@6c2cb39a[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1071387 WARN  (jetty-launcher-9930-thread-2) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@6c2cb39a[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1071390 WARN  (jetty-launcher-9930-thread-2) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@2917abde[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1071390 WARN  (jetty-launcher-9930-thread-2) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@2917abde[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1071390 INFO  (jetty-launcher-9930-thread-2) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45323/solr
   [junit4]   2> 1071391 INFO  (jetty-launcher-9930-thread-2) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1071392 INFO  (zkConnectionManagerCallback-9948-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1071392 INFO  (jetty-launcher-9930-thread-2) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1071443 INFO  (jetty-launcher-9930-thread-1) [n:127.0.0.1:45739_solr     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1071444 INFO  (zkConnectionManagerCallback-9950-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1071444 INFO  (jetty-launcher-9930-thread-1) [n:127.0.0.1:45739_solr     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1071478 INFO  (jetty-launcher-9930-thread-1) [n:127.0.0.1:45739_solr     ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:45739_solr
   [junit4]   2> 1071478 INFO  (jetty-launcher-9930-thread-1) [n:127.0.0.1:45739_solr     ] o.a.s.c.Overseer Overseer (id=72169736996323335-127.0.0.1:45739_solr-n_0000000000) starting
   [junit4]   2> 1071480 INFO  (OverseerStateUpdate-72169736996323335-127.0.0.1:45739_solr-n_0000000000) [n:127.0.0.1:45739_solr     ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:45739_solr
   [junit4]   2> 1071480 INFO  (jetty-launcher-9930-thread-1) [n:127.0.0.1:45739_solr     ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45739_solr
   [junit4]   2> 1071481 INFO  (OverseerStateUpdate-72169736996323335-127.0.0.1:45739_solr-n_0000000000) [n:127.0.0.1:45739_solr     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1071481 WARN  (jetty-launcher-9930-thread-1) [n:127.0.0.1:45739_solr     ] o.a.s.c.CoreContainer Not all security plugins configured!  authentication=disabled authorization=disabled.  Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external.  See https://s.apache.org/solrsecurity for more info
   [junit4]   2> 1071491 INFO  (jetty-launcher-9930-thread-1) [n:127.0.0.1:45739_solr     ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 1071502 INFO  (jetty-launcher-9930-thread-1) [n:127.0.0.1:45739_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'sol

[...truncated too long message...]

try=solr.jetty, tag=null
   [junit4]   2> 1073805 INFO  (jetty-closer-9969-thread-1) [     ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@7b05a4ac: rootName = solr_45739, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@5195fe2
   [junit4]   2> 1073805 INFO  (jetty-closer-9969-thread-1) [     ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 1073805 INFO  (closeThreadPool-9977-thread-2) [     ] o.a.s.c.Overseer Overseer (id=72169736996323335-127.0.0.1:45739_solr-n_0000000000) closing
   [junit4]   2> 1073805 INFO  (OverseerStateUpdate-72169736996323335-127.0.0.1:45739_solr-n_0000000000) [n:127.0.0.1:45739_solr     ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:45739_solr
   [junit4]   2> 1073805 INFO  (OverseerAutoScalingTriggerThread-72169736996323335-127.0.0.1:45739_solr-n_0000000000) [     ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 1073806 INFO  (closeThreadPool-9977-thread-1) [     ] o.a.s.c.Overseer Overseer (id=72169736996323335-127.0.0.1:45739_solr-n_0000000000) closing
   [junit4]   2> 1073898 INFO  (jetty-closer-9969-thread-2) [     ] o.e.j.s.AbstractConnector Stopped ServerConnector@201fc22e{ssl,[ssl, alpn, http/1.1, h2]}{127.0.0.1:0}
   [junit4]   2> 1073899 INFO  (jetty-closer-9969-thread-2) [     ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@4cf92585{/solr,null,UNAVAILABLE}
   [junit4]   2> 1073899 INFO  (jetty-closer-9969-thread-2) [     ] o.e.j.s.session node0 Stopped scavenging
   [junit4]   2> 1073998 INFO  (jetty-closer-9969-thread-1) [     ] o.a.s.c.Overseer Overseer (id=72169736996323335-127.0.0.1:45739_solr-n_0000000000) closing
   [junit4]   2> 1073999 INFO  (jetty-closer-9969-thread-1) [     ] o.e.j.s.AbstractConnector Stopped ServerConnector@4189c1a7{ssl,[ssl, alpn, http/1.1, h2]}{127.0.0.1:0}
   [junit4]   2> 1073999 INFO  (jetty-closer-9969-thread-1) [     ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@6b7d72a0{/solr,null,UNAVAILABLE}
   [junit4]   2> 1073999 INFO  (jetty-closer-9969-thread-1) [     ] o.e.j.s.session node0 Stopped scavenging
   [junit4]   2> 1073999 INFO  (TEST-DeleteShardTest.testDirectoryCleanupAfterDeleteShard-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.ZkTestServer Shutting down ZkTestServer.
   [junit4]   2> 1074201 WARN  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	3	/solr/aliases.json
   [junit4]   2> 	2	/solr/collections/deleteshard_test/terms/b
   [junit4]   2> 	2	/solr/collections/deleteshard_test/terms/a
   [junit4]   2> 	2	/solr/packages.json
   [junit4]   2> 	2	/solr/collections/deleteshard_test/terms/c
   [junit4]   2> 	2	/solr/security.json
   [junit4]   2> 	2	/solr/configs/conf
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	50	/solr/collections/deleteshard_test/state.json
   [junit4]   2> 	3	/solr/clusterstate.json
   [junit4]   2> 	3	/solr/clusterprops.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	27	/solr/overseer/queue
   [junit4]   2> 	12	/solr/overseer/collection-queue-work
   [junit4]   2> 	7	/solr/live_nodes
   [junit4]   2> 	6	/solr/collections
   [junit4]   2> 
   [junit4]   2> 1074203 INFO  (TEST-DeleteShardTest.testDirectoryCleanupAfterDeleteShard-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.ZkTestServer waitForServerDown: 127.0.0.1:45323
   [junit4]   2> 1074203 INFO  (TEST-DeleteShardTest.testDirectoryCleanupAfterDeleteShard-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:45323
   [junit4]   2> 1074203 INFO  (TEST-DeleteShardTest.testDirectoryCleanupAfterDeleteShard-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 45323
   [junit4]   2> 1074203 INFO  (TEST-DeleteShardTest.testDirectoryCleanupAfterDeleteShard-seed#[565BFE4F53AC2C56]) [     ] o.a.s.SolrTestCaseJ4 ###Ending testDirectoryCleanupAfterDeleteShard
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DeleteShardTest_565BFE4F53AC2C56-001
   [junit4]   2> Dec 12, 2019 7:26:07 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=HighCompressionCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=HIGH_COMPRESSION, chunkSize=1, maxDocsPerChunk=3, blockSize=671), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=HIGH_COMPRESSION, chunkSize=1, blockSize=671)), sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@147afbc5), locale=ee, timezone=Singapore
   [junit4]   2> NOTE: Linux 5.0.0-36-generic amd64/Oracle Corporation 14-ea (64-bit)/cpus=8,threads=1,free=291255656,total=497025024
   [junit4]   2> NOTE: All tests run in this JVM: [MergeStrategyTest, TestDynamicFieldCollectionResource, TestExactStatsCache, SolrMetricManagerTest, TestDefaultStatsCache, TestCloudSchemaless, TestConfig, CreateCollectionCleanupTest, SplitShardTest, DistribDocExpirationUpdateProcessorTest, TestLegacyFieldCache, TestQuerySenderNoQuery, NodeMutatorTest, SmileWriterTest, ChangedSchemaMergeTest, OverseerStatusTest, CdcrVersionReplicationTest, SearchRateTriggerTest, JsonLoaderTest, SolrRequestParserTest, TestCollectionsAPIViaSolrCloudCluster, AutoAddReplicasPlanActionTest, FileUtilsTest, HighlighterConfigTest, UpdateLogTest, AddSchemaFieldsUpdateProcessorFactoryTest, TestPerFieldSimilarityWithDefaultOverride, HdfsDirectoryFactoryTest, PathHierarchyTokenizerFactoryTest, TestConfigSetProperties, ByteBuffersDirectoryFactoryTest, PKIAuthenticationIntegrationTest, MultiSolrCloudTestCaseTest, TestPullReplica, TestDocBasedVersionConstraints, TestExportTool, CollectionsAPIAsyncDistributedZkTest, TestSchemaNameResource, ClassificationUpdateProcessorIntegrationTest, DeleteLastCustomShardedReplicaTest, TestSolrCloudWithDelegationTokens, TestFieldSortValues, TestAuthenticationFramework, HdfsLockFactoryTest, TestDistributedMap, CustomHighlightComponentTest, MetricUtilsTest, TestDynamicURP, AtomicUpdatesTest, RAMDirectoryFactoryTest, V2ApiIntegrationTest, MetricTriggerIntegrationTest, PackageManagerCLITest, TestCustomDocTransformer, UUIDFieldTest, MoveReplicaHDFSFailoverTest, TestRandomFlRTGCloud, TestStressCloudBlindAtomicUpdates, RecoveryZkTest, TestTrieFacet, ScheduledTriggerIntegrationTest, RequestLoggingTest, TestSolrQueryParser, SortSpecParsingTest, TestConfigSetsAPI, TestRequestForwarding, TestCloudRecovery2, TestMacroExpander, TestNumericRangeQuery64, SolrCLIZkUtilsTest, TestMiniSolrCloudClusterSSL, TestReplicationHandlerDiskOverFlow, TestSimNodeAddedTrigger, DistanceFunctionTest, HttpPartitionOnCommitTest, BaseCdcrDistributedZkTest, TestTestInjection, TestRandomCollapseQParserPlugin, SuggestComponentTest, TestScoreJoinQPScore, TaggingAttributeTest, TestRestManager, TestObjectReleaseTracker, TestBackupRepositoryFactory, TestSolrQueryResponse, TestSortableTextField, TestJavabinTupleStreamParser, BasicAuthStandaloneTest, TestReload, TestIBSimilarityFactory, TermsComponentTest, DocValuesMultiTest, ChaosMonkeyNothingIsSafeTest, TestMinMaxOnMultiValuedField, TestHttpShardHandlerFactory, TestLegacyTerms, NodeMarkersRegistrationTest, SimpleCollectionCreateDeleteTest, CollectionPropsTest, TestDeprecatedFilters, TestCaffeineCache, TestUtilizeNode, TestFoldingMultitermQuery, TestSuggestSpellingConverter, TestSQLHandler, AddReplicaTest, TestPKIAuthenticationPlugin, SpellCheckCollatorWithCollapseTest, PeerSyncWithBufferUpdatesTest, TestAnalyzeInfixSuggestions, TestFieldResource, TestCoreAdminApis, TestHighFrequencyDictionaryFactory, DisMaxRequestHandlerTest, TestSimpleTextCodec, TestSolrCLIRunExample, NodeAddedTriggerTest, TestFaceting, ManagedSchemaRoundRobinCloudTest, CdcrOpsAndBoundariesTest, TestSolr4Spatial, TestZkAclsWithHadoopAuth, TestSolrTestCaseJ4, TestCursorMarkWithoutUniqueKey, ZkCollectionPropsCachingTest, TriggerSetPropertiesIntegrationTest, LeaderFailoverAfterPartitionTest, BasicFunctionalityTest, TestStreamBody, TestUninvertingReader, TestDocSet, UUIDUpdateProcessorFallbackTest, OpenExchangeRatesOrgProviderTest, TestFieldCacheSortRandom, DeleteShardTest]
   [junit4] Completed [331/892 (1!)] on J1 in 8.02s, 2 tests, 1 error <<< FAILURES!

[...truncated 46819 lines...]
[repro] Jenkins log URL: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Linux/25146/consoleText

[repro] Revision: dce36c10e9021abf7936a0fc1f710a690f6f7543

[repro] Ant options: "-Dargs=-XX:-UseCompressedOops -XX:+UseG1GC"
[repro] Repro line:  ant test  -Dtestcase=DeleteShardTest -Dtests.method=test -Dtests.seed=565BFE4F53AC2C56 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=ee -Dtests.timezone=Singapore -Dtests.asserts=true -Dtests.file.encoding=UTF-8

[repro] JUnit rest result XML files will be moved to: ./repro-reports
[repro] ant clean

[...truncated 6 lines...]
[repro] Test suites by module:
[repro]    solr/core
[repro]       DeleteShardTest
[repro] ant compile-test

[...truncated 3332 lines...]
[repro] ant test-nocompile -Dtests.dups=5 -Dtests.maxfailures=5 -Dtests.class="*.DeleteShardTest" -Dtests.showOutput=onerror "-Dargs=-XX:-UseCompressedOops -XX:+UseG1GC" -Dtests.seed=565BFE4F53AC2C56 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=ee -Dtests.timezone=Singapore -Dtests.asserts=true -Dtests.file.encoding=UTF-8

[...truncated 80 lines...]
[repro] Failures w/original seeds:
[repro]   0/5 failed: org.apache.solr.cloud.DeleteShardTest
[repro] Exiting with code 0

[...truncated 43 lines...]

Re: [JENKINS-EA] Lucene-Solr-master-Linux (64bit/jdk-14-ea+24) - Build # 25146 - Unstable!

Posted by Chris Hostetter <ho...@fucit.org>.

The last time DeleteShardTest ever had a failure was 2018-Feb ... anybody 
have any idea what might have changed to cause this?

It didn't reproduce, so it's possible it's a timing bug -- but 20 months 
straight w/o a failure strongly suggests something recent helps trigger 
whatever the underlying problem is.



: Subject: [JENKINS-EA] Lucene-Solr-master-Linux (64bit/jdk-14-ea+24) - Build #
:     25146 - Unstable!
: 
: Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Linux/25146/
: Java: 64bit/jdk-14-ea+24 -XX:-UseCompressedOops -XX:+UseG1GC
: 
: 1 tests failed.
: FAILED:  org.apache.solr.cloud.DeleteShardTest.test
: 
: Error Message:
: Error from server at https://127.0.0.1:36547/solr: The slice: shard2 is currently active. Only non-active (or custom-hashed) slices can be deleted.
: 
: Stack Trace:
: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:36547/solr: The slice: shard2 is currently active. Only non-active (or custom-hashed) slices can be deleted.
: 	at __randomizedtesting.SeedInfo.seed([565BFE4F53AC2C56:DE0FC195FD5041AE]:0)
: 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:665)
: 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:265)
: 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
: 	at org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:368)
: 	at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:296)
: 	at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.sendRequest(BaseCloudSolrClient.java:1143)
: 	at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.requestWithRetryOnStaleState(BaseCloudSolrClient.java:906)
: 	at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.request(BaseCloudSolrClient.java:838)
: 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:207)
: 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:224)
: 	at org.apache.solr.cloud.DeleteShardTest.test(DeleteShardTest.java:85)
: 	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
: 	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
: 	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
: 	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
: 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
: 	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
: 	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
: 	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
: 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
: 	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
: 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
: 	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
: 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
: 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
: 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
: 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
: 	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
: 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
: 	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
: 	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
: 	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
: 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
: 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
: 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
: 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
: 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
: 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
: 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
: 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
: 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
: 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
: 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
: 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
: 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
: 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
: 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
: 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
: 	at java.base/java.lang.Thread.run(Thread.java:833)
: 
: 
: 
: 
: Build Log:
: [...truncated 14040 lines...]
:    [junit4] Suite: org.apache.solr.cloud.DeleteShardTest
:    [junit4]   2> 1067205 INFO  (SUITE-DeleteShardTest-seed#[565BFE4F53AC2C56]-worker) [     ] o.a.s.SolrTestCaseJ4 Created dataDir: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DeleteShardTest_565BFE4F53AC2C56-001/data-dir-128-001
:    [junit4]   2> 1067206 INFO  (SUITE-DeleteShardTest-seed#[565BFE4F53AC2C56]-worker) [     ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
:    [junit4]   2> 1067206 INFO  (SUITE-DeleteShardTest-seed#[565BFE4F53AC2C56]-worker) [     ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason="", ssl=0.0/0.0, value=0.0/0.0, clientAuth=0.0/0.0)
:    [junit4]   2> 1067207 INFO  (SUITE-DeleteShardTest-seed#[565BFE4F53AC2C56]-worker) [     ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
:    [junit4]   2> 1067210 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.SolrTestCaseJ4 ###Starting test
:    [junit4]   2> 1067211 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DeleteShardTest_565BFE4F53AC2C56-001/tempDir-001
:    [junit4]   2> 1067211 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
:    [junit4]   2> 1067211 INFO  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
:    [junit4]   2> 1067211 INFO  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer Starting server
:    [junit4]   2> 1067311 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.ZkTestServer start zk server on port:37469
:    [junit4]   2> 1067311 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.ZkTestServer waitForServerUp: 127.0.0.1:37469
:    [junit4]   2> 1067312 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:37469
:    [junit4]   2> 1067312 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 37469
:    [junit4]   2> 1067317 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
:    [junit4]   2> 1067341 INFO  (zkConnectionManagerCallback-9872-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
:    [junit4]   2> 1067341 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
:    [junit4]   2> 1067385 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
:    [junit4]   2> 1067398 INFO  (zkConnectionManagerCallback-9874-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
:    [junit4]   2> 1067398 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
:    [junit4]   2> 1067405 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
:    [junit4]   2> 1067412 INFO  (zkConnectionManagerCallback-9876-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
:    [junit4]   2> 1067412 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
:    [junit4]   2> 1067528 WARN  (jetty-launcher-9877-thread-1) [     ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
:    [junit4]   2> 1067528 INFO  (jetty-launcher-9877-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
:    [junit4]   2> 1067528 INFO  (jetty-launcher-9877-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
:    [junit4]   2> 1067528 INFO  (jetty-launcher-9877-thread-1) [     ] o.e.j.s.Server jetty-9.4.19.v20190610; built: 2019-06-10T16:30:51.723Z; git: afcf563148970e98786327af5e07c261fda175d3; jvm 14-ea+24-1107
:    [junit4]   2> 1067544 WARN  (jetty-launcher-9877-thread-2) [     ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
:    [junit4]   2> 1067544 INFO  (jetty-launcher-9877-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
:    [junit4]   2> 1067544 INFO  (jetty-launcher-9877-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
:    [junit4]   2> 1067544 INFO  (jetty-launcher-9877-thread-2) [     ] o.e.j.s.Server jetty-9.4.19.v20190610; built: 2019-06-10T16:30:51.723Z; git: afcf563148970e98786327af5e07c261fda175d3; jvm 14-ea+24-1107
:    [junit4]   2> 1067606 INFO  (jetty-launcher-9877-thread-1) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
:    [junit4]   2> 1067606 INFO  (jetty-launcher-9877-thread-1) [     ] o.e.j.s.session No SessionScavenger set, using defaults
:    [junit4]   2> 1067606 INFO  (jetty-launcher-9877-thread-1) [     ] o.e.j.s.session node0 Scavenging every 660000ms
:    [junit4]   2> 1067614 INFO  (jetty-launcher-9877-thread-1) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3800d98c{/solr,null,AVAILABLE}
:    [junit4]   2> 1067615 INFO  (jetty-launcher-9877-thread-1) [     ] o.e.j.s.AbstractConnector Started ServerConnector@348d3a88{ssl,[ssl, alpn, http/1.1, h2]}{127.0.0.1:36547}
:    [junit4]   2> 1067615 INFO  (jetty-launcher-9877-thread-1) [     ] o.e.j.s.Server Started @1067650ms
:    [junit4]   2> 1067615 INFO  (jetty-launcher-9877-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=36547}
:    [junit4]   2> 1067615 ERROR (jetty-launcher-9877-thread-1) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
:    [junit4]   2> 1067615 INFO  (jetty-launcher-9877-thread-1) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
:    [junit4]   2> 1067615 INFO  (jetty-launcher-9877-thread-1) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 9.0.0
:    [junit4]   2> 1067615 INFO  (jetty-launcher-9877-thread-1) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
:    [junit4]   2> 1067615 INFO  (jetty-launcher-9877-thread-1) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
:    [junit4]   2> 1067616 INFO  (jetty-launcher-9877-thread-1) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-12-12T07:26:00.712005Z
:    [junit4]   2> 1067618 INFO  (jetty-launcher-9877-thread-2) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
:    [junit4]   2> 1067618 INFO  (jetty-launcher-9877-thread-2) [     ] o.e.j.s.session No SessionScavenger set, using defaults
:    [junit4]   2> 1067618 INFO  (jetty-launcher-9877-thread-2) [     ] o.e.j.s.session node0 Scavenging every 660000ms
:    [junit4]   2> 1067626 INFO  (jetty-launcher-9877-thread-2) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@21671b0f{/solr,null,AVAILABLE}
:    [junit4]   2> 1067626 INFO  (jetty-launcher-9877-thread-1) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
:    [junit4]   2> 1067627 INFO  (jetty-launcher-9877-thread-2) [     ] o.e.j.s.AbstractConnector Started ServerConnector@25f90ff0{ssl,[ssl, alpn, http/1.1, h2]}{127.0.0.1:33457}
:    [junit4]   2> 1067627 INFO  (jetty-launcher-9877-thread-2) [     ] o.e.j.s.Server Started @1067662ms
:    [junit4]   2> 1067627 INFO  (jetty-launcher-9877-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=33457}
:    [junit4]   2> 1067627 ERROR (jetty-launcher-9877-thread-2) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
:    [junit4]   2> 1067627 INFO  (jetty-launcher-9877-thread-2) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
:    [junit4]   2> 1067627 INFO  (jetty-launcher-9877-thread-2) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 9.0.0
:    [junit4]   2> 1067627 INFO  (jetty-launcher-9877-thread-2) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
:    [junit4]   2> 1067627 INFO  (jetty-launcher-9877-thread-2) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
:    [junit4]   2> 1067627 INFO  (jetty-launcher-9877-thread-2) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-12-12T07:26:00.723439Z
:    [junit4]   2> 1067630 INFO  (zkConnectionManagerCallback-9879-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
:    [junit4]   2> 1067630 INFO  (jetty-launcher-9877-thread-1) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
:    [junit4]   2> 1067630 INFO  (jetty-launcher-9877-thread-1) [     ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
:    [junit4]   2> 1067653 INFO  (jetty-launcher-9877-thread-2) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
:    [junit4]   2> 1067671 INFO  (zkConnectionManagerCallback-9881-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
:    [junit4]   2> 1067671 INFO  (jetty-launcher-9877-thread-2) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
:    [junit4]   2> 1067672 INFO  (jetty-launcher-9877-thread-2) [     ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
:    [junit4]   2> 1068012 INFO  (jetty-launcher-9877-thread-2) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
:    [junit4]   2> 1068013 WARN  (jetty-launcher-9877-thread-2) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@5dcccdb4[provider=null,keyStore=null,trustStore=null]
:    [junit4]   2> 1068013 WARN  (jetty-launcher-9877-thread-2) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@5dcccdb4[provider=null,keyStore=null,trustStore=null]
:    [junit4]   2> 1068021 WARN  (jetty-launcher-9877-thread-2) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@7f256270[provider=null,keyStore=null,trustStore=null]
:    [junit4]   2> 1068021 WARN  (jetty-launcher-9877-thread-2) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@7f256270[provider=null,keyStore=null,trustStore=null]
:    [junit4]   2> 1068022 INFO  (jetty-launcher-9877-thread-2) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:37469/solr
:    [junit4]   2> 1068024 INFO  (jetty-launcher-9877-thread-2) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
:    [junit4]   2> 1068024 INFO  (zkConnectionManagerCallback-9889-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
:    [junit4]   2> 1068024 INFO  (jetty-launcher-9877-thread-2) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
:    [junit4]   2> 1068112 INFO  (jetty-launcher-9877-thread-1) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
:    [junit4]   2> 1068113 WARN  (jetty-launcher-9877-thread-1) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@742042e0[provider=null,keyStore=null,trustStore=null]
:    [junit4]   2> 1068113 WARN  (jetty-launcher-9877-thread-1) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@742042e0[provider=null,keyStore=null,trustStore=null]
:    [junit4]   2> 1068122 WARN  (jetty-launcher-9877-thread-1) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@1862647a[provider=null,keyStore=null,trustStore=null]
:    [junit4]   2> 1068122 WARN  (jetty-launcher-9877-thread-1) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@1862647a[provider=null,keyStore=null,trustStore=null]
:    [junit4]   2> 1068123 INFO  (jetty-launcher-9877-thread-1) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:37469/solr
:    [junit4]   2> 1068152 INFO  (jetty-launcher-9877-thread-1) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
:    [junit4]   2> 1068152 INFO  (jetty-launcher-9877-thread-2) [n:127.0.0.1:33457_solr     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
:    [junit4]   2> 1068154 INFO  (zkConnectionManagerCallback-9895-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
:    [junit4]   2> 1068154 INFO  (jetty-launcher-9877-thread-1) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
:    [junit4]   2> 1068162 INFO  (zkConnectionManagerCallback-9897-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
:    [junit4]   2> 1068162 INFO  (jetty-launcher-9877-thread-2) [n:127.0.0.1:33457_solr     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
:    [junit4]   2> 1068214 INFO  (jetty-launcher-9877-thread-2) [n:127.0.0.1:33457_solr     ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:33457_solr
:    [junit4]   2> 1068215 INFO  (jetty-launcher-9877-thread-2) [n:127.0.0.1:33457_solr     ] o.a.s.c.Overseer Overseer (id=72169736756330503-127.0.0.1:33457_solr-n_0000000000) starting
:    [junit4]   2> 1068217 INFO  (OverseerStateUpdate-72169736756330503-127.0.0.1:33457_solr-n_0000000000) [n:127.0.0.1:33457_solr     ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:33457_solr
:    [junit4]   2> 1068218 INFO  (jetty-launcher-9877-thread-2) [n:127.0.0.1:33457_solr     ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33457_solr
:    [junit4]   2> 1068219 WARN  (jetty-launcher-9877-thread-2) [n:127.0.0.1:33457_solr     ] o.a.s.c.CoreContainer Not all security plugins configured!  authentication=disabled authorization=disabled.  Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external.  See https://s.apache.org/solrsecurity for more info
:    [junit4]   2> 1068222 INFO  (zkCallback-9896-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
:    [junit4]   2> 1068234 INFO  (jetty-launcher-9877-thread-2) [n:127.0.0.1:33457_solr     ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
:    [junit4]   2> 1068249 INFO  (jetty-launcher-9877-thread-2) [n:127.0.0.1:33457_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33457.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@61231b3f
:    [junit4]   2> 1068255 INFO  (jetty-launcher-9877-thread-2) [n:127.0.0.1:33457_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33457.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@61231b3f
:    [junit4]   2> 1068255 INFO  (jetty-launcher-9877-thread-2) [n:127.0.0.1:33457_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33457.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@61231b3f
:    [junit4]   2> 1068256 INFO  (jetty-launcher-9877-thread-1) [n:127.0.0.1:36547_solr     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
:    [junit4]   2> 1068256 INFO  (jetty-launcher-9877-thread-2) [n:127.0.0.1:33457_solr     ] o.a.s.p.PackageLoader /packages.json updated to version -1
:    [junit4]   2> 1068256 INFO  (zkConnectionManagerCallback-9903-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
:    [junit4]   2> 1068256 INFO  (jetty-launcher-9877-thread-1) [n:127.0.0.1:36547_solr     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
:    [junit4]   2> 1068257 INFO  (jetty-launcher-9877-thread-2) [n:127.0.0.1:33457_solr     ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DeleteShardTest_565BFE4F53AC2C56-001/tempDir-001/node2/.
:    [junit4]   2> 1068259 INFO  (jetty-launcher-9877-thread-1) [n:127.0.0.1:36547_solr     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
:    [junit4]   2> 1068261 INFO  (jetty-launcher-9877-thread-1) [n:127.0.0.1:36547_solr     ] o.a.s.c.ZkController Publish node=127.0.0.1:36547_solr as DOWN
:    [junit4]   2> 1068262 INFO  (jetty-launcher-9877-thread-1) [n:127.0.0.1:36547_solr     ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
:    [junit4]   2> 1068262 INFO  (jetty-launcher-9877-thread-1) [n:127.0.0.1:36547_solr     ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:36547_solr
:    [junit4]   2> 1068262 INFO  (zkCallback-9896-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
:    [junit4]   2> 1068262 INFO  (zkCallback-9902-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
:    [junit4]   2> 1068263 WARN  (jetty-launcher-9877-thread-1) [n:127.0.0.1:36547_solr     ] o.a.s.c.CoreContainer Not all security plugins configured!  authentication=disabled authorization=disabled.  Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external.  See https://s.apache.org/solrsecurity for more info
:    [junit4]   2> 1068269 INFO  (jetty-launcher-9877-thread-1) [n:127.0.0.1:36547_solr     ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
:    [junit4]   2> 1068284 INFO  (jetty-launcher-9877-thread-1) [n:127.0.0.1:36547_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_36547.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@61231b3f
:    [junit4]   2> 1068294 INFO  (jetty-launcher-9877-thread-1) [n:127.0.0.1:36547_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_36547.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@61231b3f
:    [junit4]   2> 1068294 INFO  (jetty-launcher-9877-thread-1) [n:127.0.0.1:36547_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_36547.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@61231b3f
:    [junit4]   2> 1068300 INFO  (jetty-launcher-9877-thread-1) [n:127.0.0.1:36547_solr     ] o.a.s.p.PackageLoader /packages.json updated to version -1
:    [junit4]   2> 1068301 INFO  (jetty-launcher-9877-thread-1) [n:127.0.0.1:36547_solr     ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DeleteShardTest_565BFE4F53AC2C56-001/tempDir-001/node1/.
:    [junit4]   2> 1068318 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.MiniSolrCloudCluster waitForAllNodes: numServers=2
:    [junit4]   2> 1068319 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
:    [junit4]   2> 1068320 INFO  (zkConnectionManagerCallback-9912-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
:    [junit4]   2> 1068320 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
:    [junit4]   2> 1068322 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
:    [junit4]   2> 1068323 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:37469/solr ready
:    [junit4]   2> 1068339 INFO  (qtp2071450029-22167) [n:127.0.0.1:36547_solr     ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf&name=deleteShard&nrtReplicas=1&action=CREATE&numShards=2&wt=javabin&version=2 and sendToOCPQueue=true
:    [junit4]   2> 1068341 INFO  (OverseerThreadFactory-6026-thread-1-processing-n:127.0.0.1:33457_solr) [n:127.0.0.1:33457_solr     ] o.a.s.c.a.c.CreateCollectionCmd Create collection deleteShard
:    [junit4]   2> 1068446 INFO  (OverseerStateUpdate-72169736756330503-127.0.0.1:33457_solr-n_0000000000) [n:127.0.0.1:33457_solr     ] o.a.s.c.o.SliceMutator createReplica() {
:    [junit4]   2>   "operation":"ADDREPLICA",
:    [junit4]   2>   "collection":"deleteShard",
:    [junit4]   2>   "shard":"shard1",
:    [junit4]   2>   "core":"deleteShard_shard1_replica_n1",
:    [junit4]   2>   "state":"down",
:    [junit4]   2>   "base_url":"https://127.0.0.1:33457/solr",
:    [junit4]   2>   "type":"NRT",
:    [junit4]   2>   "waitForFinalState":"false"} 
:    [junit4]   2> 1068447 INFO  (OverseerStateUpdate-72169736756330503-127.0.0.1:33457_solr-n_0000000000) [n:127.0.0.1:33457_solr     ] o.a.s.c.o.SliceMutator createReplica() {
:    [junit4]   2>   "operation":"ADDREPLICA",
:    [junit4]   2>   "collection":"deleteShard",
:    [junit4]   2>   "shard":"shard2",
:    [junit4]   2>   "core":"deleteShard_shard2_replica_n2",
:    [junit4]   2>   "state":"down",
:    [junit4]   2>   "base_url":"https://127.0.0.1:36547/solr",
:    [junit4]   2>   "type":"NRT",
:    [junit4]   2>   "waitForFinalState":"false"} 
:    [junit4]   2> 1068658 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr    x:deleteShard_shard2_replica_n2 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=deleteShard_shard2_replica_n2&action=CREATE&numShards=2&collection=deleteShard&shard=shard2&wt=javabin&version=2&replicaType=NRT
:    [junit4]   2> 1068661 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr    x:deleteShard_shard1_replica_n1 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=deleteShard_shard1_replica_n1&action=CREATE&numShards=2&collection=deleteShard&shard=shard1&wt=javabin&version=2&replicaType=NRT
:    [junit4]   2> 1068662 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr    x:deleteShard_shard1_replica_n1 ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
:    [junit4]   2> 1069668 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.0.0
:    [junit4]   2> 1069670 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.0.0
:    [junit4]   2> 1069674 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.s.IndexSchema [deleteShard_shard2_replica_n2] Schema name=minimal
:    [junit4]   2> 1069675 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
:    [junit4]   2> 1069675 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.c.CoreContainer Creating SolrCore 'deleteShard_shard2_replica_n2' using configuration from collection deleteShard, trusted=true
:    [junit4]   2> 1069676 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_36547.solr.core.deleteShard.shard2.replica_n2' (registry 'solr.core.deleteShard.shard2.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@61231b3f
:    [junit4]   2> 1069677 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.s.IndexSchema [deleteShard_shard1_replica_n1] Schema name=minimal
:    [junit4]   2> 1069677 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.c.SolrCore [[deleteShard_shard2_replica_n2] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DeleteShardTest_565BFE4F53AC2C56-001/tempDir-001/node1/deleteShard_shard2_replica_n2], dataDir=[/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DeleteShardTest_565BFE4F53AC2C56-001/tempDir-001/node1/./deleteShard_shard2_replica_n2/data/]
:    [junit4]   2> 1069678 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
:    [junit4]   2> 1069678 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.c.CoreContainer Creating SolrCore 'deleteShard_shard1_replica_n1' using configuration from collection deleteShard, trusted=true
:    [junit4]   2> 1069679 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33457.solr.core.deleteShard.shard1.replica_n1' (registry 'solr.core.deleteShard.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@61231b3f
:    [junit4]   2> 1069679 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.c.SolrCore [[deleteShard_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DeleteShardTest_565BFE4F53AC2C56-001/tempDir-001/node2/deleteShard_shard1_replica_n1], dataDir=[/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DeleteShardTest_565BFE4F53AC2C56-001/tempDir-001/node2/./deleteShard_shard1_replica_n1/data/]
:    [junit4]   2> 1069705 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
:    [junit4]   2> 1069705 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
:    [junit4]   2> 1069705 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
:    [junit4]   2> 1069705 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
:    [junit4]   2> 1069707 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.u.CommitTracker Hard AutoCommit: disabled
:    [junit4]   2> 1069707 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.u.CommitTracker Hard AutoCommit: disabled
:    [junit4]   2> 1069707 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
:    [junit4]   2> 1069707 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
:    [junit4]   2> 1069707 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.s.SolrIndexSearcher Opening [Searcher@63cd4fe2[deleteShard_shard1_replica_n1] main]
:    [junit4]   2> 1069707 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.s.SolrIndexSearcher Opening [Searcher@1654af20[deleteShard_shard2_replica_n2] main]
:    [junit4]   2> 1069708 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
:    [junit4]   2> 1069708 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
:    [junit4]   2> 1069708 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
:    [junit4]   2> 1069708 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
:    [junit4]   2> 1069709 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
:    [junit4]   2> 1069709 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1652697923903815680
:    [junit4]   2> 1069709 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
:    [junit4]   2> 1069709 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1652697923903815680
:    [junit4]   2> 1069711 INFO  (searcherExecutor-6035-thread-1-processing-n:127.0.0.1:36547_solr x:deleteShard_shard2_replica_n2 c:deleteShard s:shard2 r:core_node4) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.c.SolrCore [deleteShard_shard2_replica_n2] Registered new searcher Searcher@1654af20[deleteShard_shard2_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
:    [junit4]   2> 1069711 INFO  (searcherExecutor-6036-thread-1-processing-n:127.0.0.1:33457_solr x:deleteShard_shard1_replica_n1 c:deleteShard s:shard1 r:core_node3) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.c.SolrCore [deleteShard_shard1_replica_n1] Registered new searcher Searcher@63cd4fe2[deleteShard_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
:    [junit4]   2> 1069713 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.c.ZkShardTerms Successful update of terms at /collections/deleteShard/terms/shard2 to Terms{values={core_node4=0}, version=0}
:    [junit4]   2> 1069713 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/deleteShard/leaders/shard2
:    [junit4]   2> 1069714 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.c.ZkShardTerms Successful update of terms at /collections/deleteShard/terms/shard1 to Terms{values={core_node3=0}, version=0}
:    [junit4]   2> 1069714 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/deleteShard/leaders/shard1
:    [junit4]   2> 1069715 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
:    [junit4]   2> 1069715 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
:    [junit4]   2> 1069715 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:36547/solr/deleteShard_shard2_replica_n2/
:    [junit4]   2> 1069718 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
:    [junit4]   2> 1069718 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.c.SyncStrategy https://127.0.0.1:36547/solr/deleteShard_shard2_replica_n2/ has no replicas
:    [junit4]   2> 1069718 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/deleteShard/leaders/shard2/leader after winning as /collections/deleteShard/leader_elect/shard2/election/72169736756330504-core_node4-n_0000000000
:    [junit4]   2> 1069719 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:36547/solr/deleteShard_shard2_replica_n2/ shard2
:    [junit4]   2> 1069721 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
:    [junit4]   2> 1069721 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
:    [junit4]   2> 1069721 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:33457/solr/deleteShard_shard1_replica_n1/
:    [junit4]   2> 1069721 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
:    [junit4]   2> 1069721 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.c.SyncStrategy https://127.0.0.1:33457/solr/deleteShard_shard1_replica_n1/ has no replicas
:    [junit4]   2> 1069721 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/deleteShard/leaders/shard1/leader after winning as /collections/deleteShard/leader_elect/shard1/election/72169736756330503-core_node3-n_0000000000
:    [junit4]   2> 1069722 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:33457/solr/deleteShard_shard1_replica_n1/ shard1
:    [junit4]   2> 1069823 INFO  (zkCallback-9902-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
:    [junit4]   2> 1069823 INFO  (zkCallback-9896-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
:    [junit4]   2> 1069824 INFO  (zkCallback-9902-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
:    [junit4]   2> 1069824 INFO  (zkCallback-9896-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
:    [junit4]   2> 1069824 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.c.ZkController I am the leader, no recovery necessary
:    [junit4]   2> 1069825 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.c.ZkController I am the leader, no recovery necessary
:    [junit4]   2> 1069827 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=deleteShard_shard2_replica_n2&action=CREATE&numShards=2&collection=deleteShard&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1169
:    [junit4]   2> 1069827 INFO  (qtp1778644731-22169) [n:127.0.0.1:33457_solr c:deleteShard s:shard1 r:core_node3 x:deleteShard_shard1_replica_n1 ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=deleteShard_shard1_replica_n1&action=CREATE&numShards=2&collection=deleteShard&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1165
:    [junit4]   2> 1069828 INFO  (qtp2071450029-22167) [n:127.0.0.1:36547_solr     ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
:    [junit4]   2> 1069927 INFO  (zkCallback-9902-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
:    [junit4]   2> 1069927 INFO  (zkCallback-9902-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
:    [junit4]   2> 1069927 INFO  (zkCallback-9896-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
:    [junit4]   2> 1069927 INFO  (zkCallback-9896-thread-3) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
:    [junit4]   2> 1069927 INFO  (zkCallback-9902-thread-3) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
:    [junit4]   2> 1069928 INFO  (zkCallback-9896-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
:    [junit4]   2> 1069928 INFO  (qtp2071450029-22167) [n:127.0.0.1:36547_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf&name=deleteShard&nrtReplicas=1&action=CREATE&numShards=2&wt=javabin&version=2} status=0 QTime=1589
:    [junit4]   2> 1069928 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.MiniSolrCloudCluster waitForActiveCollection: deleteShard
:    [junit4]   2> 1069935 INFO  (qtp2071450029-22165) [n:127.0.0.1:36547_solr     ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :deleteshard with params action=DELETESHARD&collection=deleteShard&shard=shard1&wt=javabin&version=2 and sendToOCPQueue=true
:    [junit4]   2> 1069937 INFO  (OverseerThreadFactory-6026-thread-2-processing-n:127.0.0.1:33457_solr) [n:127.0.0.1:33457_solr c:deleteShard s:shard1   ] o.a.s.c.a.c.DeleteShardCmd Delete shard invoked
:    [junit4]   2> 1069938 ERROR (OverseerThreadFactory-6026-thread-2-processing-n:127.0.0.1:33457_solr) [n:127.0.0.1:33457_solr c:deleteShard s:shard1   ] o.a.s.c.a.c.OverseerCollectionMessageHandler Collection: deleteShard operation: deleteshard failed:org.apache.solr.common.SolrException: The slice: shard1 is currently active. Only non-active (or custom-hashed) slices can be deleted.
:    [junit4]   2> 	at org.apache.solr.cloud.api.collections.DeleteShardCmd.call(DeleteShardCmd.java:87)
:    [junit4]   2> 	at org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler.processMessage(OverseerCollectionMessageHandler.java:263)
:    [junit4]   2> 	at org.apache.solr.cloud.OverseerTaskProcessor$Runner.run(OverseerTaskProcessor.java:505)
:    [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
:    [junit4]   2> 	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
:    [junit4]   2> 	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
:    [junit4]   2> 	at java.base/java.lang.Thread.run(Thread.java:833)
:    [junit4]   2> 
:    [junit4]   2> 1069938 INFO  (OverseerCollectionConfigSetProcessor-72169736756330503-127.0.0.1:33457_solr-n_0000000000) [n:127.0.0.1:33457_solr     ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
:    [junit4]   2> 1069941 INFO  (qtp2071450029-22165) [n:127.0.0.1:36547_solr c:deleteShard    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=DELETESHARD&collection=deleteShard&shard=shard1&wt=javabin&version=2} status=400 QTime=6
:    [junit4]   2> 1069943 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.SolrCloudTestCase waitForState (deleteShard): Expected shard shard1 to be in state inactive
:    [junit4]   2> 1069943 INFO  (OverseerStateUpdate-72169736756330503-127.0.0.1:33457_solr-n_0000000000) [n:127.0.0.1:33457_solr     ] o.a.s.c.o.SliceMutator Update shard state invoked for collection: deleteShard with message: {
:    [junit4]   2>   "shard1":"inactive",
:    [junit4]   2>   "collection":"deleteShard",
:    [junit4]   2>   "operation":"updateshardstate"}
:    [junit4]   2> 1069943 INFO  (OverseerStateUpdate-72169736756330503-127.0.0.1:33457_solr-n_0000000000) [n:127.0.0.1:33457_solr     ] o.a.s.c.o.SliceMutator Update shard state shard1 to inactive
:    [junit4]   2> 1070044 INFO  (zkCallback-9896-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
:    [junit4]   2> 1070044 INFO  (zkCallback-9911-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
:    [junit4]   2> 1070044 INFO  (zkCallback-9896-thread-3) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
:    [junit4]   2> 1070044 INFO  (zkCallback-9896-thread-4) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
:    [junit4]   2> 1070044 INFO  (zkCallback-9902-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
:    [junit4]   2> 1070044 INFO  (zkCallback-9902-thread-3) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
:    [junit4]   2> 1070044 INFO  (zkCallback-9902-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
:    [junit4]   2> 1070044 INFO  (zkCallback-9911-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
:    [junit4]   2> 1070045 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr     ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :deleteshard with params action=DELETESHARD&collection=deleteShard&shard=shard1&wt=javabin&version=2 and sendToOCPQueue=true
:    [junit4]   2> 1070046 INFO  (OverseerThreadFactory-6026-thread-3-processing-n:127.0.0.1:33457_solr) [n:127.0.0.1:33457_solr c:deleteShard s:shard1   ] o.a.s.c.a.c.DeleteShardCmd Delete shard invoked
:    [junit4]   2> 1070046 INFO  (OverseerThreadFactory-6026-thread-3-processing-n:127.0.0.1:33457_solr) [n:127.0.0.1:33457_solr c:deleteShard s:shard1   ] o.a.s.c.a.c.DeleteShardCmd Deleting replica for collection=deleteShard shard=shard1 on node=127.0.0.1:33457_solr
:    [junit4]   2> 1070046 INFO  (OverseerCollectionConfigSetProcessor-72169736756330503-127.0.0.1:33457_solr-n_0000000000) [n:127.0.0.1:33457_solr     ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist.  Requestor may have disconnected from ZooKeeper
:    [junit4]   2> 1070047 INFO  (qtp1778644731-22166) [n:127.0.0.1:33457_solr    x:deleteShard_shard1_replica_n1 ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.deleteShard.shard1.replica_n1, tag=null
:    [junit4]   2> 1070047 INFO  (qtp1778644731-22166) [n:127.0.0.1:33457_solr    x:deleteShard_shard1_replica_n1 ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@67edf51: rootName = solr_33457, domain = solr.core.deleteShard.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.deleteShard.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@48925ba4
:    [junit4]   2> 1070060 INFO  (qtp1778644731-22166) [n:127.0.0.1:33457_solr    x:deleteShard_shard1_replica_n1 ] o.a.s.c.SolrCore [deleteShard_shard1_replica_n1]  CLOSING SolrCore org.apache.solr.core.SolrCore@1106cd65
:    [junit4]   2> 1070060 INFO  (qtp1778644731-22166) [n:127.0.0.1:33457_solr    x:deleteShard_shard1_replica_n1 ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.deleteShard.shard1.replica_n1, tag=SolrCore@1106cd65
:    [junit4]   2> 1070060 INFO  (qtp1778644731-22166) [n:127.0.0.1:33457_solr    x:deleteShard_shard1_replica_n1 ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.deleteShard.shard1.leader, tag=SolrCore@1106cd65
:    [junit4]   2> 1070061 INFO  (qtp1778644731-22166) [n:127.0.0.1:33457_solr    x:deleteShard_shard1_replica_n1 ] o.a.s.c.ZkShardTerms Successful update of terms at /collections/deleteShard/terms/shard1 to Terms{values={}, version=1}
:    [junit4]   2> 1070062 INFO  (qtp1778644731-22166) [n:127.0.0.1:33457_solr    x:deleteShard_shard1_replica_n1 ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={deleteInstanceDir=true&core=deleteShard_shard1_replica_n1&deleteMetricsHistory=true&qt=/admin/cores&deleteDataDir=true&action=UNLOAD&wt=javabin&version=2&deleteIndex=true} status=0 QTime=15
:    [junit4]   2> 1070163 INFO  (zkCallback-9902-thread-3) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
:    [junit4]   2> 1070163 INFO  (zkCallback-9902-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
:    [junit4]   2> 1070163 INFO  (zkCallback-9902-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
:    [junit4]   2> 1070163 INFO  (zkCallback-9896-thread-4) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
:    [junit4]   2> 1070163 INFO  (zkCallback-9896-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
:    [junit4]   2> 1070163 INFO  (zkCallback-9896-thread-3) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
:    [junit4]   2> 1070164 INFO  (OverseerStateUpdate-72169736756330503-127.0.0.1:33457_solr-n_0000000000) [n:127.0.0.1:33457_solr     ] o.a.s.c.o.CollectionMutator Removing collection: deleteShard shard: shard1 from clusterstate
:    [junit4]   2> 1070265 INFO  (zkCallback-9896-thread-3) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
:    [junit4]   2> 1070265 INFO  (zkCallback-9896-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
:    [junit4]   2> 1070265 INFO  (zkCallback-9896-thread-4) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
:    [junit4]   2> 1070265 INFO  (zkCallback-9896-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
:    [junit4]   2> 1070265 INFO  (zkCallback-9902-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
:    [junit4]   2> 1070265 INFO  (zkCallback-9902-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
:    [junit4]   2> 1070265 INFO  (zkCallback-9902-thread-3) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
:    [junit4]   2> 1070266 INFO  (OverseerThreadFactory-6026-thread-3-processing-n:127.0.0.1:33457_solr) [n:127.0.0.1:33457_solr c:deleteShard s:shard1   ] o.a.s.c.a.c.DeleteShardCmd Successfully deleted collection: deleteShard, shard: shard1
:    [junit4]   2> 1070266 INFO  (qtp2071450029-22168) [n:127.0.0.1:36547_solr c:deleteShard    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=DELETESHARD&collection=deleteShard&shard=shard1&wt=javabin&version=2} status=0 QTime=221
:    [junit4]   2> 1070267 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.SolrCloudTestCase waitForState (deleteShard): Expected 'shard1' to be removed
:    [junit4]   2> 1070268 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.SolrCloudTestCase waitForState (deleteShard): Expected shard shard2 to be in state construction
:    [junit4]   2> 1070268 INFO  (OverseerStateUpdate-72169736756330503-127.0.0.1:33457_solr-n_0000000000) [n:127.0.0.1:33457_solr     ] o.a.s.c.o.SliceMutator Update shard state invoked for collection: deleteShard with message: {
:    [junit4]   2>   "shard2":"construction",
:    [junit4]   2>   "collection":"deleteShard",
:    [junit4]   2>   "operation":"updateshardstate"}
:    [junit4]   2> 1070268 INFO  (OverseerStateUpdate-72169736756330503-127.0.0.1:33457_solr-n_0000000000) [n:127.0.0.1:33457_solr     ] o.a.s.c.o.SliceMutator Update shard state shard2 to construction
:    [junit4]   2> 1070369 INFO  (zkCallback-9902-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
:    [junit4]   2> 1070369 INFO  (zkCallback-9911-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
:    [junit4]   2> 1070369 INFO  (zkCallback-9902-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
:    [junit4]   2> 1070369 INFO  (zkCallback-9902-thread-3) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
:    [junit4]   2> 1070369 INFO  (zkCallback-9911-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteShard/state.json] for collection [deleteShard] has occurred - updating... (live nodes size: [2])
:    [junit4]   2> 1070370 INFO  (qtp2071450029-22167) [n:127.0.0.1:36547_solr     ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :deleteshard with params action=DELETESHARD&collection=deleteShard&shard=shard2&wt=javabin&version=2 and sendToOCPQueue=true
:    [junit4]   2> 1070371 INFO  (OverseerThreadFactory-6026-thread-4-processing-n:127.0.0.1:33457_solr) [n:127.0.0.1:33457_solr c:deleteShard s:shard2   ] o.a.s.c.a.c.DeleteShardCmd Delete shard invoked
:    [junit4]   2> 1070371 ERROR (OverseerThreadFactory-6026-thread-4-processing-n:127.0.0.1:33457_solr) [n:127.0.0.1:33457_solr c:deleteShard s:shard2   ] o.a.s.c.a.c.OverseerCollectionMessageHandler Collection: deleteShard operation: deleteshard failed:org.apache.solr.common.SolrException: The slice: shard2 is currently active. Only non-active (or custom-hashed) slices can be deleted.
:    [junit4]   2> 	at org.apache.solr.cloud.api.collections.DeleteShardCmd.call(DeleteShardCmd.java:87)
:    [junit4]   2> 	at org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler.processMessage(OverseerCollectionMessageHandler.java:263)
:    [junit4]   2> 	at org.apache.solr.cloud.OverseerTaskProcessor$Runner.run(OverseerTaskProcessor.java:505)
:    [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
:    [junit4]   2> 	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
:    [junit4]   2> 	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
:    [junit4]   2> 	at java.base/java.lang.Thread.run(Thread.java:833)
:    [junit4]   2> 
:    [junit4]   2> 1070371 INFO  (OverseerCollectionConfigSetProcessor-72169736756330503-127.0.0.1:33457_solr-n_0000000000) [n:127.0.0.1:33457_solr     ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist.  Requestor may have disconnected from ZooKeeper
:    [junit4]   2> 1070373 INFO  (qtp2071450029-22167) [n:127.0.0.1:36547_solr c:deleteShard    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=DELETESHARD&collection=deleteShard&shard=shard2&wt=javabin&version=2} status=400 QTime=3
:    [junit4]   2> 1070475 INFO  (jetty-closer-9915-thread-1) [     ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1320324481
:    [junit4]   2> 1070475 INFO  (jetty-closer-9915-thread-1) [     ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:33457_solr
:    [junit4]   2> 1070476 INFO  (jetty-closer-9915-thread-1) [     ] o.a.s.c.ZkController Publish this node as DOWN...
:    [junit4]   2> 1070476 INFO  (jetty-closer-9915-thread-1) [     ] o.a.s.c.ZkController Publish node=127.0.0.1:33457_solr as DOWN
:    [junit4]   2> 1070476 INFO  (jetty-closer-9915-thread-2) [     ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=134538154
:    [junit4]   2> 1070476 INFO  (jetty-closer-9915-thread-2) [     ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:36547_solr
:    [junit4]   2> 1070476 INFO  (jetty-closer-9915-thread-2) [     ] o.a.s.c.ZkController Publish this node as DOWN...
:    [junit4]   2> 1070476 INFO  (jetty-closer-9915-thread-2) [     ] o.a.s.c.ZkController Publish node=127.0.0.1:36547_solr as DOWN
:    [junit4]   2> 1070477 INFO  (jetty-closer-9915-thread-1) [     ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
:    [junit4]   2> 1070477 INFO  (jetty-closer-9915-thread-1) [     ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@4446e372: rootName = solr_33457, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@81cf508
:    [junit4]   2> 1070481 INFO  (jetty-closer-9915-thread-1) [     ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
:    [junit4]   2> 1070481 INFO  (jetty-closer-9915-thread-1) [     ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@fef8ab6: rootName = solr_33457, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@39d8c401
:    [junit4]   2> 1070482 INFO  (coreCloseExecutor-6046-thread-1) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.c.SolrCore [deleteShard_shard2_replica_n2]  CLOSING SolrCore org.apache.solr.core.SolrCore@77976d35
:    [junit4]   2> 1070482 INFO  (coreCloseExecutor-6046-thread-1) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.deleteShard.shard2.replica_n2, tag=SolrCore@77976d35
:    [junit4]   2> 1070482 INFO  (coreCloseExecutor-6046-thread-1) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@359201bd: rootName = solr_36547, domain = solr.core.deleteShard.shard2.replica_n2, service url = null, agent id = null] for registry solr.core.deleteShard.shard2.replica_n2 / com.codahale.metrics.MetricRegistry@594f48a7
:    [junit4]   2> 1070482 INFO  (jetty-closer-9915-thread-1) [     ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
:    [junit4]   2> 1070482 INFO  (jetty-closer-9915-thread-1) [     ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@2a570f0b: rootName = solr_33457, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@5195fe2
:    [junit4]   2> 1070483 INFO  (jetty-closer-9915-thread-1) [     ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
:    [junit4]   2> 1070483 INFO  (closeThreadPool-9922-thread-2) [     ] o.a.s.c.Overseer Overseer (id=72169736756330503-127.0.0.1:33457_solr-n_0000000000) closing
:    [junit4]   2> 1070483 INFO  (OverseerStateUpdate-72169736756330503-127.0.0.1:33457_solr-n_0000000000) [n:127.0.0.1:33457_solr     ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:33457_solr
:    [junit4]   2> 1070483 INFO  (OverseerAutoScalingTriggerThread-72169736756330503-127.0.0.1:33457_solr-n_0000000000) [     ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
:    [junit4]   2> 1070486 INFO  (closeThreadPool-9922-thread-1) [     ] o.a.s.c.Overseer Overseer (id=72169736756330503-127.0.0.1:33457_solr-n_0000000000) closing
:    [junit4]   2> 1070486 INFO  (zkCallback-9902-thread-3) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
:    [junit4]   2> 1070490 INFO  (coreCloseExecutor-6046-thread-1) [n:127.0.0.1:36547_solr c:deleteShard s:shard2 r:core_node4 x:deleteShard_shard2_replica_n2 ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.deleteShard.shard2.leader, tag=SolrCore@77976d35
:    [junit4]   2> 1070491 INFO  (jetty-closer-9915-thread-2) [     ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
:    [junit4]   2> 1070491 INFO  (jetty-closer-9915-thread-2) [     ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@29cd5ae1: rootName = solr_36547, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@562f42d6
:    [junit4]   2> 1070495 INFO  (jetty-closer-9915-thread-2) [     ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
:    [junit4]   2> 1070495 INFO  (jetty-closer-9915-thread-2) [     ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@53d4764b: rootName = solr_36547, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@39d8c401
:    [junit4]   2> 1070497 INFO  (jetty-closer-9915-thread-2) [     ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
:    [junit4]   2> 1070497 INFO  (jetty-closer-9915-thread-2) [     ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@146125e0: rootName = solr_36547, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@5195fe2
:    [junit4]   2> 1070497 INFO  (jetty-closer-9915-thread-2) [     ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
:    [junit4]   2> 1070586 INFO  (jetty-closer-9915-thread-1) [     ] o.a.s.c.Overseer Overseer (id=72169736756330503-127.0.0.1:33457_solr-n_0000000000) closing
:    [junit4]   2> 1070587 INFO  (jetty-closer-9915-thread-1) [     ] o.e.j.s.AbstractConnector Stopped ServerConnector@25f90ff0{ssl,[ssl, alpn, http/1.1, h2]}{127.0.0.1:0}
:    [junit4]   2> 1070587 INFO  (jetty-closer-9915-thread-1) [     ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@21671b0f{/solr,null,UNAVAILABLE}
:    [junit4]   2> 1070587 INFO  (jetty-closer-9915-thread-1) [     ] o.e.j.s.session node0 Stopped scavenging
:    [junit4]   2> 1070687 INFO  (jetty-closer-9915-thread-2) [     ] o.e.j.s.AbstractConnector Stopped ServerConnector@348d3a88{ssl,[ssl, alpn, http/1.1, h2]}{127.0.0.1:0}
:    [junit4]   2> 1070687 INFO  (jetty-closer-9915-thread-2) [     ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@3800d98c{/solr,null,UNAVAILABLE}
:    [junit4]   2> 1070688 INFO  (jetty-closer-9915-thread-2) [     ] o.e.j.s.session node0 Stopped scavenging
:    [junit4]   2> 1070688 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.ZkTestServer Shutting down ZkTestServer.
:    [junit4]   2> 1070890 WARN  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer Watch limit violations: 
:    [junit4]   2> Maximum concurrent create/delete watches above limit:
:    [junit4]   2> 
:    [junit4]   2> 	3	/solr/collections/deleteShard/terms/shard1
:    [junit4]   2> 	3	/solr/aliases.json
:    [junit4]   2> 	2	/solr/packages.json
:    [junit4]   2> 	2	/solr/security.json
:    [junit4]   2> 	2	/solr/configs/conf
:    [junit4]   2> 	2	/solr/collections/deleteShard/terms/shard2
:    [junit4]   2> 
:    [junit4]   2> Maximum concurrent data watches above limit:
:    [junit4]   2> 
:    [junit4]   2> 	49	/solr/collections/deleteShard/state.json
:    [junit4]   2> 	3	/solr/clusterstate.json
:    [junit4]   2> 	3	/solr/clusterprops.json
:    [junit4]   2> 
:    [junit4]   2> Maximum concurrent children watches above limit:
:    [junit4]   2> 
:    [junit4]   2> 	25	/solr/overseer/queue
:    [junit4]   2> 	15	/solr/overseer/collection-queue-work
:    [junit4]   2> 	7	/solr/live_nodes
:    [junit4]   2> 	6	/solr/collections
:    [junit4]   2> 
:    [junit4]   2> 1070891 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.ZkTestServer waitForServerDown: 127.0.0.1:37469
:    [junit4]   2> 1070891 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:37469
:    [junit4]   2> 1070891 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 37469
:    [junit4]   2> 1070892 INFO  (TEST-DeleteShardTest.test-seed#[565BFE4F53AC2C56]) [     ] o.a.s.SolrTestCaseJ4 ###Ending test
:    [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=DeleteShardTest -Dtests.method=test -Dtests.seed=565BFE4F53AC2C56 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=ee -Dtests.timezone=Singapore -Dtests.asserts=true -Dtests.file.encoding=UTF-8
:    [junit4] ERROR   3.69s J1 | DeleteShardTest.test <<<
:    [junit4]    > Throwable #1: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:36547/solr: The slice: shard2 is currently active. Only non-active (or custom-hashed) slices can be deleted.
:    [junit4]    > 	at __randomizedtesting.SeedInfo.seed([565BFE4F53AC2C56:DE0FC195FD5041AE]:0)
:    [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:665)
:    [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:265)
:    [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
:    [junit4]    > 	at org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:368)
:    [junit4]    > 	at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:296)
:    [junit4]    > 	at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.sendRequest(BaseCloudSolrClient.java:1143)
:    [junit4]    > 	at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.requestWithRetryOnStaleState(BaseCloudSolrClient.java:906)
:    [junit4]    > 	at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.request(BaseCloudSolrClient.java:838)
:    [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:207)
:    [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:224)
:    [junit4]    > 	at org.apache.solr.cloud.DeleteShardTest.test(DeleteShardTest.java:85)
:    [junit4]    > 	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
:    [junit4]    > 	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
:    [junit4]    > 	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
:    [junit4]    > 	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
:    [junit4]    > 	at java.base/java.lang.Thread.run(Thread.java:833)
:    [junit4]   2> 1070895 INFO  (TEST-DeleteShardTest.testDirectoryCleanupAfterDeleteShard-seed#[565BFE4F53AC2C56]) [     ] o.a.s.SolrTestCaseJ4 ###Starting testDirectoryCleanupAfterDeleteShard
:    [junit4]   2> 1070895 INFO  (TEST-DeleteShardTest.testDirectoryCleanupAfterDeleteShard-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DeleteShardTest_565BFE4F53AC2C56-001/tempDir-002
:    [junit4]   2> 1070896 INFO  (TEST-DeleteShardTest.testDirectoryCleanupAfterDeleteShard-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
:    [junit4]   2> 1070896 INFO  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
:    [junit4]   2> 1070896 INFO  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer Starting server
:    [junit4]   2> 1070996 INFO  (TEST-DeleteShardTest.testDirectoryCleanupAfterDeleteShard-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.ZkTestServer start zk server on port:45323
:    [junit4]   2> 1070996 INFO  (TEST-DeleteShardTest.testDirectoryCleanupAfterDeleteShard-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.ZkTestServer waitForServerUp: 127.0.0.1:45323
:    [junit4]   2> 1070996 INFO  (TEST-DeleteShardTest.testDirectoryCleanupAfterDeleteShard-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:45323
:    [junit4]   2> 1070996 INFO  (TEST-DeleteShardTest.testDirectoryCleanupAfterDeleteShard-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 45323
:    [junit4]   2> 1070997 INFO  (TEST-DeleteShardTest.testDirectoryCleanupAfterDeleteShard-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
:    [junit4]   2> 1070998 INFO  (zkConnectionManagerCallback-9925-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
:    [junit4]   2> 1070998 INFO  (TEST-DeleteShardTest.testDirectoryCleanupAfterDeleteShard-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
:    [junit4]   2> 1071000 INFO  (TEST-DeleteShardTest.testDirectoryCleanupAfterDeleteShard-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
:    [junit4]   2> 1071001 INFO  (zkConnectionManagerCallback-9927-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
:    [junit4]   2> 1071001 INFO  (TEST-DeleteShardTest.testDirectoryCleanupAfterDeleteShard-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
:    [junit4]   2> 1071001 INFO  (TEST-DeleteShardTest.testDirectoryCleanupAfterDeleteShard-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
:    [junit4]   2> 1071002 INFO  (zkConnectionManagerCallback-9929-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
:    [junit4]   2> 1071002 INFO  (TEST-DeleteShardTest.testDirectoryCleanupAfterDeleteShard-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
:    [junit4]   2> 1071105 WARN  (jetty-launcher-9930-thread-1) [     ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
:    [junit4]   2> 1071106 INFO  (jetty-launcher-9930-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
:    [junit4]   2> 1071106 INFO  (jetty-launcher-9930-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
:    [junit4]   2> 1071106 INFO  (jetty-launcher-9930-thread-1) [     ] o.e.j.s.Server jetty-9.4.19.v20190610; built: 2019-06-10T16:30:51.723Z; git: afcf563148970e98786327af5e07c261fda175d3; jvm 14-ea+24-1107
:    [junit4]   2> 1071106 WARN  (jetty-launcher-9930-thread-2) [     ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
:    [junit4]   2> 1071106 INFO  (jetty-launcher-9930-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
:    [junit4]   2> 1071106 INFO  (jetty-launcher-9930-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
:    [junit4]   2> 1071106 INFO  (jetty-launcher-9930-thread-2) [     ] o.e.j.s.Server jetty-9.4.19.v20190610; built: 2019-06-10T16:30:51.723Z; git: afcf563148970e98786327af5e07c261fda175d3; jvm 14-ea+24-1107
:    [junit4]   2> 1071108 INFO  (jetty-launcher-9930-thread-1) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
:    [junit4]   2> 1071108 INFO  (jetty-launcher-9930-thread-1) [     ] o.e.j.s.session No SessionScavenger set, using defaults
:    [junit4]   2> 1071108 INFO  (jetty-launcher-9930-thread-1) [     ] o.e.j.s.session node0 Scavenging every 660000ms
:    [junit4]   2> 1071110 INFO  (jetty-launcher-9930-thread-1) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6b7d72a0{/solr,null,AVAILABLE}
:    [junit4]   2> 1071115 INFO  (jetty-launcher-9930-thread-1) [     ] o.e.j.s.AbstractConnector Started ServerConnector@4189c1a7{ssl,[ssl, alpn, http/1.1, h2]}{127.0.0.1:45739}
:    [junit4]   2> 1071115 INFO  (jetty-launcher-9930-thread-1) [     ] o.e.j.s.Server Started @1071150ms
:    [junit4]   2> 1071115 INFO  (jetty-launcher-9930-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=45739}
:    [junit4]   2> 1071116 ERROR (jetty-launcher-9930-thread-1) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
:    [junit4]   2> 1071116 INFO  (jetty-launcher-9930-thread-1) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
:    [junit4]   2> 1071116 INFO  (jetty-launcher-9930-thread-1) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 9.0.0
:    [junit4]   2> 1071116 INFO  (jetty-launcher-9930-thread-1) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
:    [junit4]   2> 1071116 INFO  (jetty-launcher-9930-thread-1) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
:    [junit4]   2> 1071116 INFO  (jetty-launcher-9930-thread-1) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-12-12T07:26:04.212145Z
:    [junit4]   2> 1071124 INFO  (jetty-launcher-9930-thread-1) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
:    [junit4]   2> 1071124 INFO  (jetty-launcher-9930-thread-2) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
:    [junit4]   2> 1071124 INFO  (jetty-launcher-9930-thread-2) [     ] o.e.j.s.session No SessionScavenger set, using defaults
:    [junit4]   2> 1071124 INFO  (jetty-launcher-9930-thread-2) [     ] o.e.j.s.session node0 Scavenging every 660000ms
:    [junit4]   2> 1071125 INFO  (zkConnectionManagerCallback-9932-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
:    [junit4]   2> 1071125 INFO  (jetty-launcher-9930-thread-2) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4cf92585{/solr,null,AVAILABLE}
:    [junit4]   2> 1071125 INFO  (jetty-launcher-9930-thread-1) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
:    [junit4]   2> 1071126 INFO  (jetty-launcher-9930-thread-1) [     ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
:    [junit4]   2> 1071126 INFO  (jetty-launcher-9930-thread-2) [     ] o.e.j.s.AbstractConnector Started ServerConnector@201fc22e{ssl,[ssl, alpn, http/1.1, h2]}{127.0.0.1:37321}
:    [junit4]   2> 1071126 INFO  (jetty-launcher-9930-thread-2) [     ] o.e.j.s.Server Started @1071161ms
:    [junit4]   2> 1071126 INFO  (jetty-launcher-9930-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=37321}
:    [junit4]   2> 1071127 ERROR (jetty-launcher-9930-thread-2) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
:    [junit4]   2> 1071127 INFO  (jetty-launcher-9930-thread-2) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
:    [junit4]   2> 1071127 INFO  (jetty-launcher-9930-thread-2) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 9.0.0
:    [junit4]   2> 1071127 INFO  (jetty-launcher-9930-thread-2) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
:    [junit4]   2> 1071127 INFO  (jetty-launcher-9930-thread-2) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
:    [junit4]   2> 1071127 INFO  (jetty-launcher-9930-thread-2) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-12-12T07:26:04.223096Z
:    [junit4]   2> 1071130 INFO  (jetty-launcher-9930-thread-2) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
:    [junit4]   2> 1071131 INFO  (zkConnectionManagerCallback-9934-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
:    [junit4]   2> 1071131 INFO  (jetty-launcher-9930-thread-2) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
:    [junit4]   2> 1071131 INFO  (jetty-launcher-9930-thread-2) [     ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
:    [junit4]   2> 1071326 INFO  (jetty-launcher-9930-thread-1) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
:    [junit4]   2> 1071327 WARN  (jetty-launcher-9930-thread-1) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@45e5d0a5[provider=null,keyStore=null,trustStore=null]
:    [junit4]   2> 1071327 WARN  (jetty-launcher-9930-thread-1) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@45e5d0a5[provider=null,keyStore=null,trustStore=null]
:    [junit4]   2> 1071336 WARN  (jetty-launcher-9930-thread-1) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@4656cc40[provider=null,keyStore=null,trustStore=null]
:    [junit4]   2> 1071336 WARN  (jetty-launcher-9930-thread-1) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@4656cc40[provider=null,keyStore=null,trustStore=null]
:    [junit4]   2> 1071337 INFO  (jetty-launcher-9930-thread-1) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45323/solr
:    [junit4]   2> 1071338 INFO  (jetty-launcher-9930-thread-1) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
:    [junit4]   2> 1071341 INFO  (zkConnectionManagerCallback-9941-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
:    [junit4]   2> 1071342 INFO  (jetty-launcher-9930-thread-1) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
:    [junit4]   2> 1071386 INFO  (jetty-launcher-9930-thread-2) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
:    [junit4]   2> 1071387 WARN  (jetty-launcher-9930-thread-2) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@6c2cb39a[provider=null,keyStore=null,trustStore=null]
:    [junit4]   2> 1071387 WARN  (jetty-launcher-9930-thread-2) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@6c2cb39a[provider=null,keyStore=null,trustStore=null]
:    [junit4]   2> 1071390 WARN  (jetty-launcher-9930-thread-2) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@2917abde[provider=null,keyStore=null,trustStore=null]
:    [junit4]   2> 1071390 WARN  (jetty-launcher-9930-thread-2) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@2917abde[provider=null,keyStore=null,trustStore=null]
:    [junit4]   2> 1071390 INFO  (jetty-launcher-9930-thread-2) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45323/solr
:    [junit4]   2> 1071391 INFO  (jetty-launcher-9930-thread-2) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
:    [junit4]   2> 1071392 INFO  (zkConnectionManagerCallback-9948-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
:    [junit4]   2> 1071392 INFO  (jetty-launcher-9930-thread-2) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
:    [junit4]   2> 1071443 INFO  (jetty-launcher-9930-thread-1) [n:127.0.0.1:45739_solr     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
:    [junit4]   2> 1071444 INFO  (zkConnectionManagerCallback-9950-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
:    [junit4]   2> 1071444 INFO  (jetty-launcher-9930-thread-1) [n:127.0.0.1:45739_solr     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
:    [junit4]   2> 1071478 INFO  (jetty-launcher-9930-thread-1) [n:127.0.0.1:45739_solr     ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:45739_solr
:    [junit4]   2> 1071478 INFO  (jetty-launcher-9930-thread-1) [n:127.0.0.1:45739_solr     ] o.a.s.c.Overseer Overseer (id=72169736996323335-127.0.0.1:45739_solr-n_0000000000) starting
:    [junit4]   2> 1071480 INFO  (OverseerStateUpdate-72169736996323335-127.0.0.1:45739_solr-n_0000000000) [n:127.0.0.1:45739_solr     ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:45739_solr
:    [junit4]   2> 1071480 INFO  (jetty-launcher-9930-thread-1) [n:127.0.0.1:45739_solr     ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45739_solr
:    [junit4]   2> 1071481 INFO  (OverseerStateUpdate-72169736996323335-127.0.0.1:45739_solr-n_0000000000) [n:127.0.0.1:45739_solr     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
:    [junit4]   2> 1071481 WARN  (jetty-launcher-9930-thread-1) [n:127.0.0.1:45739_solr     ] o.a.s.c.CoreContainer Not all security plugins configured!  authentication=disabled authorization=disabled.  Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external.  See https://s.apache.org/solrsecurity for more info
:    [junit4]   2> 1071491 INFO  (jetty-launcher-9930-thread-1) [n:127.0.0.1:45739_solr     ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
:    [junit4]   2> 1071502 INFO  (jetty-launcher-9930-thread-1) [n:127.0.0.1:45739_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'sol
: 
: [...truncated too long message...]
: 
: try=solr.jetty, tag=null
:    [junit4]   2> 1073805 INFO  (jetty-closer-9969-thread-1) [     ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@7b05a4ac: rootName = solr_45739, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@5195fe2
:    [junit4]   2> 1073805 INFO  (jetty-closer-9969-thread-1) [     ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
:    [junit4]   2> 1073805 INFO  (closeThreadPool-9977-thread-2) [     ] o.a.s.c.Overseer Overseer (id=72169736996323335-127.0.0.1:45739_solr-n_0000000000) closing
:    [junit4]   2> 1073805 INFO  (OverseerStateUpdate-72169736996323335-127.0.0.1:45739_solr-n_0000000000) [n:127.0.0.1:45739_solr     ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:45739_solr
:    [junit4]   2> 1073805 INFO  (OverseerAutoScalingTriggerThread-72169736996323335-127.0.0.1:45739_solr-n_0000000000) [     ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
:    [junit4]   2> 1073806 INFO  (closeThreadPool-9977-thread-1) [     ] o.a.s.c.Overseer Overseer (id=72169736996323335-127.0.0.1:45739_solr-n_0000000000) closing
:    [junit4]   2> 1073898 INFO  (jetty-closer-9969-thread-2) [     ] o.e.j.s.AbstractConnector Stopped ServerConnector@201fc22e{ssl,[ssl, alpn, http/1.1, h2]}{127.0.0.1:0}
:    [junit4]   2> 1073899 INFO  (jetty-closer-9969-thread-2) [     ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@4cf92585{/solr,null,UNAVAILABLE}
:    [junit4]   2> 1073899 INFO  (jetty-closer-9969-thread-2) [     ] o.e.j.s.session node0 Stopped scavenging
:    [junit4]   2> 1073998 INFO  (jetty-closer-9969-thread-1) [     ] o.a.s.c.Overseer Overseer (id=72169736996323335-127.0.0.1:45739_solr-n_0000000000) closing
:    [junit4]   2> 1073999 INFO  (jetty-closer-9969-thread-1) [     ] o.e.j.s.AbstractConnector Stopped ServerConnector@4189c1a7{ssl,[ssl, alpn, http/1.1, h2]}{127.0.0.1:0}
:    [junit4]   2> 1073999 INFO  (jetty-closer-9969-thread-1) [     ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@6b7d72a0{/solr,null,UNAVAILABLE}
:    [junit4]   2> 1073999 INFO  (jetty-closer-9969-thread-1) [     ] o.e.j.s.session node0 Stopped scavenging
:    [junit4]   2> 1073999 INFO  (TEST-DeleteShardTest.testDirectoryCleanupAfterDeleteShard-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.ZkTestServer Shutting down ZkTestServer.
:    [junit4]   2> 1074201 WARN  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer Watch limit violations: 
:    [junit4]   2> Maximum concurrent create/delete watches above limit:
:    [junit4]   2> 
:    [junit4]   2> 	3	/solr/aliases.json
:    [junit4]   2> 	2	/solr/collections/deleteshard_test/terms/b
:    [junit4]   2> 	2	/solr/collections/deleteshard_test/terms/a
:    [junit4]   2> 	2	/solr/packages.json
:    [junit4]   2> 	2	/solr/collections/deleteshard_test/terms/c
:    [junit4]   2> 	2	/solr/security.json
:    [junit4]   2> 	2	/solr/configs/conf
:    [junit4]   2> 
:    [junit4]   2> Maximum concurrent data watches above limit:
:    [junit4]   2> 
:    [junit4]   2> 	50	/solr/collections/deleteshard_test/state.json
:    [junit4]   2> 	3	/solr/clusterstate.json
:    [junit4]   2> 	3	/solr/clusterprops.json
:    [junit4]   2> 
:    [junit4]   2> Maximum concurrent children watches above limit:
:    [junit4]   2> 
:    [junit4]   2> 	27	/solr/overseer/queue
:    [junit4]   2> 	12	/solr/overseer/collection-queue-work
:    [junit4]   2> 	7	/solr/live_nodes
:    [junit4]   2> 	6	/solr/collections
:    [junit4]   2> 
:    [junit4]   2> 1074203 INFO  (TEST-DeleteShardTest.testDirectoryCleanupAfterDeleteShard-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.ZkTestServer waitForServerDown: 127.0.0.1:45323
:    [junit4]   2> 1074203 INFO  (TEST-DeleteShardTest.testDirectoryCleanupAfterDeleteShard-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:45323
:    [junit4]   2> 1074203 INFO  (TEST-DeleteShardTest.testDirectoryCleanupAfterDeleteShard-seed#[565BFE4F53AC2C56]) [     ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 45323
:    [junit4]   2> 1074203 INFO  (TEST-DeleteShardTest.testDirectoryCleanupAfterDeleteShard-seed#[565BFE4F53AC2C56]) [     ] o.a.s.SolrTestCaseJ4 ###Ending testDirectoryCleanupAfterDeleteShard
:    [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DeleteShardTest_565BFE4F53AC2C56-001
:    [junit4]   2> Dec 12, 2019 7:26:07 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
:    [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
:    [junit4]   2> NOTE: test params are: codec=HighCompressionCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=HIGH_COMPRESSION, chunkSize=1, maxDocsPerChunk=3, blockSize=671), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=HIGH_COMPRESSION, chunkSize=1, blockSize=671)), sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@147afbc5), locale=ee, timezone=Singapore
:    [junit4]   2> NOTE: Linux 5.0.0-36-generic amd64/Oracle Corporation 14-ea (64-bit)/cpus=8,threads=1,free=291255656,total=497025024
:    [junit4]   2> NOTE: All tests run in this JVM: [MergeStrategyTest, TestDynamicFieldCollectionResource, TestExactStatsCache, SolrMetricManagerTest, TestDefaultStatsCache, TestCloudSchemaless, TestConfig, CreateCollectionCleanupTest, SplitShardTest, DistribDocExpirationUpdateProcessorTest, TestLegacyFieldCache, TestQuerySenderNoQuery, NodeMutatorTest, SmileWriterTest, ChangedSchemaMergeTest, OverseerStatusTest, CdcrVersionReplicationTest, SearchRateTriggerTest, JsonLoaderTest, SolrRequestParserTest, TestCollectionsAPIViaSolrCloudCluster, AutoAddReplicasPlanActionTest, FileUtilsTest, HighlighterConfigTest, UpdateLogTest, AddSchemaFieldsUpdateProcessorFactoryTest, TestPerFieldSimilarityWithDefaultOverride, HdfsDirectoryFactoryTest, PathHierarchyTokenizerFactoryTest, TestConfigSetProperties, ByteBuffersDirectoryFactoryTest, PKIAuthenticationIntegrationTest, MultiSolrCloudTestCaseTest, TestPullReplica, TestDocBasedVersionConstraints, TestExportTool, CollectionsAPIAsyncDistributedZkTes
 t, TestSchemaNameResource, ClassificationUpdateProcessorIntegrationTest, DeleteLastCustomShardedReplicaTest, TestSolrCloudWithDelegationTokens, TestFieldSortValues, TestAuthenticationFramework, HdfsLockFactoryTest, TestDistributedMap, CustomHighlightComponentTest, MetricUtilsTest, TestDynamicURP, AtomicUpdatesTest, RAMDirectoryFactoryTest, V2ApiIntegrationTest, MetricTriggerIntegrationTest, PackageManagerCLITest, TestCustomDocTransformer, UUIDFieldTest, MoveReplicaHDFSFailoverTest, TestRandomFlRTGCloud, TestStressCloudBlindAtomicUpdates, RecoveryZkTest, TestTrieFacet, ScheduledTriggerIntegrationTest, RequestLoggingTest, TestSolrQueryParser, SortSpecParsingTest, TestConfigSetsAPI, TestRequestForwarding, TestCloudRecovery2, TestMacroExpander, TestNumericRangeQuery64, SolrCLIZkUtilsTest, TestMiniSolrCloudClusterSSL, TestReplicationHandlerDiskOverFlow, TestSimNodeAddedTrigger, DistanceFunctionTest, HttpPartitionOnCommitTest, BaseCdcrDistributedZkTest, TestTestInjection, TestRandomCollap
 seQParserPlugin, SuggestComponentTest, TestScoreJoinQPScore, TaggingAttributeTest, TestRestManager, TestObjectReleaseTracker, TestBackupRepositoryFactory, TestSolrQueryResponse, TestSortableTextField, TestJavabinTupleStreamParser, BasicAuthStandaloneTest, TestReload, TestIBSimilarityFactory, TermsComponentTest, DocValuesMultiTest, ChaosMonkeyNothingIsSafeTest, TestMinMaxOnMultiValuedField, TestHttpShardHandlerFactory, TestLegacyTerms, NodeMarkersRegistrationTest, SimpleCollectionCreateDeleteTest, CollectionPropsTest, TestDeprecatedFilters, TestCaffeineCache, TestUtilizeNode, TestFoldingMultitermQuery, TestSuggestSpellingConverter, TestSQLHandler, AddReplicaTest, TestPKIAuthenticationPlugin, SpellCheckCollatorWithCollapseTest, PeerSyncWithBufferUpdatesTest, TestAnalyzeInfixSuggestions, TestFieldResource, TestCoreAdminApis, TestHighFrequencyDictionaryFactory, DisMaxRequestHandlerTest, TestSimpleTextCodec, TestSolrCLIRunExample, NodeAddedTriggerTest, TestFaceting, ManagedSchemaRoundRob
 inCloudTest, CdcrOpsAndBoundariesTest, TestSolr4Spatial, TestZkAclsWithHadoopAuth, TestSolrTestCaseJ4, TestCursorMarkWithoutUniqueKey, ZkCollectionPropsCachingTest, TriggerSetPropertiesIntegrationTest, LeaderFailoverAfterPartitionTest, BasicFunctionalityTest, TestStreamBody, TestUninvertingReader, TestDocSet, UUIDUpdateProcessorFallbackTest, OpenExchangeRatesOrgProviderTest, TestFieldCacheSortRandom, DeleteShardTest]
:    [junit4] Completed [331/892 (1!)] on J1 in 8.02s, 2 tests, 1 error <<< FAILURES!
: 
: [...truncated 46819 lines...]
: [repro] Jenkins log URL: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Linux/25146/consoleText
: 
: [repro] Revision: dce36c10e9021abf7936a0fc1f710a690f6f7543
: 
: [repro] Ant options: "-Dargs=-XX:-UseCompressedOops -XX:+UseG1GC"
: [repro] Repro line:  ant test  -Dtestcase=DeleteShardTest -Dtests.method=test -Dtests.seed=565BFE4F53AC2C56 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=ee -Dtests.timezone=Singapore -Dtests.asserts=true -Dtests.file.encoding=UTF-8
: 
: [repro] JUnit rest result XML files will be moved to: ./repro-reports
: [repro] ant clean
: 
: [...truncated 6 lines...]
: [repro] Test suites by module:
: [repro]    solr/core
: [repro]       DeleteShardTest
: [repro] ant compile-test
: 
: [...truncated 3332 lines...]
: [repro] ant test-nocompile -Dtests.dups=5 -Dtests.maxfailures=5 -Dtests.class="*.DeleteShardTest" -Dtests.showOutput=onerror "-Dargs=-XX:-UseCompressedOops -XX:+UseG1GC" -Dtests.seed=565BFE4F53AC2C56 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=ee -Dtests.timezone=Singapore -Dtests.asserts=true -Dtests.file.encoding=UTF-8
: 
: [...truncated 80 lines...]
: [repro] Failures w/original seeds:
: [repro]   0/5 failed: org.apache.solr.cloud.DeleteShardTest
: [repro] Exiting with code 0
: 
: [...truncated 43 lines...]
: 

-Hoss
http://www.lucidworks.com/