You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2018/03/15 16:09:56 UTC

[JENKINS] Lucene-Solr-master-Solaris (64bit/jdk1.8.0) - Build # 1734 - Failure!

Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1734/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseG1GC

1 tests failed.
FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
Didn't see all replicas for shard shard1 in collection1 come up within 30000 ms! ClusterState: {   "control_collection":{     "pullReplicas":"0",     "replicationFactor":"1",     "shards":{"shard1":{         "range":"80000000-7fffffff",         "state":"active",         "replicas":{"core_node2":{             "core":"control_collection_shard1_replica_n1",             "base_url":"http://127.0.0.1:62648/dl/ew",             "node_name":"127.0.0.1:62648_dl%2Few",             "state":"active",             "type":"NRT",             "leader":"true"}}}},     "router":{"name":"compositeId"},     "maxShardsPerNode":"1",     "autoAddReplicas":"false",     "nrtReplicas":"1",     "tlogReplicas":"0"},   "collection1":{     "pullReplicas":"0",     "replicationFactor":"1",     "shards":{"shard1":{         "range":"80000000-7fffffff",         "state":"active",         "replicas":{           "core_node22":{             "core":"collection1_shard1_replica_t21",             "base_url":"http://127.0.0.1:52399/dl/ew",             "node_name":"127.0.0.1:52399_dl%2Few",             "state":"active",             "type":"TLOG",             "leader":"true"},           "core_node24":{             "core":"collection1_shard1_replica_t23",             "base_url":"http://127.0.0.1:64943/dl/ew",             "node_name":"127.0.0.1:64943_dl%2Few",             "state":"recovering",             "type":"TLOG"}}}},     "router":{"name":"compositeId"},     "maxShardsPerNode":"1",     "autoAddReplicas":"false",     "nrtReplicas":"1",     "tlogReplicas":"0"}}

Stack Trace:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in collection1 come up within 30000 ms! ClusterState: {
  "control_collection":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "base_url":"http://127.0.0.1:62648/dl/ew",
            "node_name":"127.0.0.1:62648_dl%2Few",
            "state":"active",
            "type":"NRT",
            "leader":"true"}}}},
    "router":{"name":"compositeId"},
    "maxShardsPerNode":"1",
    "autoAddReplicas":"false",
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "collection1":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node22":{
            "core":"collection1_shard1_replica_t21",
            "base_url":"http://127.0.0.1:52399/dl/ew",
            "node_name":"127.0.0.1:52399_dl%2Few",
            "state":"active",
            "type":"TLOG",
            "leader":"true"},
          "core_node24":{
            "core":"collection1_shard1_replica_t23",
            "base_url":"http://127.0.0.1:64943/dl/ew",
            "node_name":"127.0.0.1:64943_dl%2Few",
            "state":"recovering",
            "type":"TLOG"}}}},
    "router":{"name":"compositeId"},
    "maxShardsPerNode":"1",
    "autoAddReplicas":"false",
    "nrtReplicas":"1",
    "tlogReplicas":"0"}}
	at __randomizedtesting.SeedInfo.seed([95A7A0DC3B52F81B:1DF39F0695AE95E3]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.ensureAllReplicasAreActive(AbstractFullDistribZkTestBase.java:2005)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:116)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 1846 lines...]
   [junit4] JVM J0: stdout was not empty, see: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/lucene/build/core/test/temp/junit4-J0-20180315_143733_9541180553243220927029.sysout
   [junit4] >>> JVM J0 emitted unexpected output (verbatim) ----
   [junit4] codec: Asserting, pf: FSTOrd50, dvf: Lucene70
   [junit4] <<< JVM J0: EOF ----

[...truncated 12180 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest_95A7A0DC3B52F81B-001/init-core-data-001
   [junit4]   2> 2538330 WARN  (SUITE-RecoveryAfterSoftCommitTest-seed#[95A7A0DC3B52F81B]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=6 numCloses=6
   [junit4]   2> 2538331 INFO  (SUITE-RecoveryAfterSoftCommitTest-seed#[95A7A0DC3B52F81B]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 2538332 INFO  (SUITE-RecoveryAfterSoftCommitTest-seed#[95A7A0DC3B52F81B]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=None)
   [junit4]   2> 2538332 INFO  (SUITE-RecoveryAfterSoftCommitTest-seed#[95A7A0DC3B52F81B]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 2538332 INFO  (SUITE-RecoveryAfterSoftCommitTest-seed#[95A7A0DC3B52F81B]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /dl/ew
   [junit4]   2> 2538334 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 2538334 INFO  (Thread-5019) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 2538335 INFO  (Thread-5019) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 2538337 ERROR (Thread-5019) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 2538434 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.c.ZkTestServer start zk server on port:54034
   [junit4]   2> 2538453 INFO  (zkConnectionManagerCallback-5320-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2538466 INFO  (zkConnectionManagerCallback-5322-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2538475 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 2538517 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 2538521 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 2538523 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 2538525 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 2538526 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 2538528 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 2538530 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 2538531 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 2538533 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 2538534 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 2538536 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Will use TLOG replicas unless explicitly asked otherwise
   [junit4]   2> 2538677 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T16:27:37-05:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 2538678 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 2538679 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 2538679 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.e.j.s.session Scavenging every 660000ms
   [junit4]   2> 2538679 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@39bea704{/dl/ew,null,AVAILABLE}
   [junit4]   2> 2538679 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@3fda0bc9{HTTP/1.1,[http/1.1]}{127.0.0.1:50816}
   [junit4]   2> 2538679 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.e.j.s.Server Started @2542206ms
   [junit4]   2> 2538679 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest_95A7A0DC3B52F81B-001/tempDir-001/control/data, replicaType=NRT, hostContext=/dl/ew, hostPort=62648, coreRootDirectory=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest_95A7A0DC3B52F81B-001/control-001/cores}
   [junit4]   2> 2538680 ERROR (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2538680 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 2538680 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2538680 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2538680 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-03-15T15:36:55.209Z
   [junit4]   2> 2538682 INFO  (zkConnectionManagerCallback-5324-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2538683 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 2538683 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest_95A7A0DC3B52F81B-001/control-001/solr.xml
   [junit4]   2> 2538686 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 2538686 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 2538687 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@b81b1b4, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 2538689 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:54034/solr
   [junit4]   2> 2538691 INFO  (zkConnectionManagerCallback-5328-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2538694 INFO  (zkConnectionManagerCallback-5330-thread-1-processing-n:127.0.0.1:62648_dl%2Few) [n:127.0.0.1:62648_dl%2Few    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2538855 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:62648_dl%2Few    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2538858 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:62648_dl%2Few    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:62648_dl%2Few
   [junit4]   2> 2538863 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:62648_dl%2Few    ] o.a.s.c.Overseer Overseer (id=72064476004220932-127.0.0.1:62648_dl%2Few-n_0000000000) starting
   [junit4]   2> 2538873 INFO  (zkConnectionManagerCallback-5335-thread-1-processing-n:127.0.0.1:62648_dl%2Few) [n:127.0.0.1:62648_dl%2Few    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2538875 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:62648_dl%2Few    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:54034/solr ready
   [junit4]   2> 2538877 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:62648_dl%2Few    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:62648_dl%2Few
   [junit4]   2> 2538881 INFO  (zkCallback-5329-thread-1-processing-n:127.0.0.1:62648_dl%2Few) [n:127.0.0.1:62648_dl%2Few    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2538882 INFO  (zkCallback-5334-thread-1-processing-n:127.0.0.1:62648_dl%2Few) [n:127.0.0.1:62648_dl%2Few    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2539075 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:62648_dl%2Few    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b81b1b4
   [junit4]   2> 2539082 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:62648_dl%2Few    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b81b1b4
   [junit4]   2> 2539082 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:62648_dl%2Few    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b81b1b4
   [junit4]   2> 2539083 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:62648_dl%2Few    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest_95A7A0DC3B52F81B-001/control-001/cores
   [junit4]   2> 2539104 INFO  (zkConnectionManagerCallback-5339-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2539106 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2539108 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:54034/solr ready
   [junit4]   2> 2539108 INFO  (SocketProxy-Acceptor-62648) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=46834,localport=62648], receiveBufferSize:130880
   [junit4]   2> 2539109 INFO  (SocketProxy-Acceptor-62648) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=50816,localport=45296], receiveBufferSize=139264
   [junit4]   2> 2539110 INFO  (qtp686307169-24869) [n:127.0.0.1:62648_dl%2Few    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:62648_dl%252Few&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2539113 INFO  (OverseerThreadFactory-9898-thread-1-processing-n:127.0.0.1:62648_dl%2Few) [n:127.0.0.1:62648_dl%2Few    ] o.a.s.c.a.c.CreateCollectionCmd Create collection control_collection
   [junit4]   2> 2539223 INFO  (SocketProxy-Acceptor-62648) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=36484,localport=62648], receiveBufferSize:130880
   [junit4]   2> 2539224 INFO  (SocketProxy-Acceptor-62648) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=50816,localport=39435], receiveBufferSize=139264
   [junit4]   2> 2539224 INFO  (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 2539225 INFO  (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 2539332 INFO  (zkCallback-5329-thread-1-processing-n:127.0.0.1:62648_dl%2Few) [n:127.0.0.1:62648_dl%2Few    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 2540256 INFO  (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 2540279 INFO  (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
   [junit4]   2> 2540420 INFO  (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 2540466 INFO  (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
   [junit4]   2> 2540467 INFO  (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b81b1b4
   [junit4]   2> 2540467 INFO  (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 2540467 INFO  (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest_95A7A0DC3B52F81B-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest_95A7A0DC3B52F81B-001/control-001/cores/control_collection_shard1_replica_n1/data/]
   [junit4]   2> 2540470 INFO  (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=28, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=4.8076171875, floorSegmentMB=1.9384765625, forceMergeDeletesPctAllowed=22.775887864813843, segmentsPerTier=13.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.31703850642037046
   [junit4]   2> 2540474 WARN  (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 2540528 INFO  (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2540528 INFO  (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=2 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2540529 INFO  (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2540529 INFO  (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2540530 INFO  (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=22, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 2540530 INFO  (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@22a6a530[control_collection_shard1_replica_n1] main]
   [junit4]   2> 2540536 INFO  (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 2540537 INFO  (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2540537 INFO  (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 2540538 INFO  (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1595018541339246592
   [junit4]   2> 2540540 INFO  (searcherExecutor-9901-thread-1-processing-n:127.0.0.1:62648_dl%2Few x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@22a6a530[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2540550 INFO  (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkShardTerms Successful update of terms at /collections/control_collection/terms/shard1 to Terms{values={core_node2=0}, version=0}
   [junit4]   2> 2540556 INFO  (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 2540556 INFO  (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 2540556 INFO  (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:62648/dl/ew/control_collection_shard1_replica_n1/
   [junit4]   2> 2540557 INFO  (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 2540557 INFO  (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:62648/dl/ew/control_collection_shard1_replica_n1/ has no replicas
   [junit4]   2> 2540557 INFO  (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 2540566 INFO  (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:62648/dl/ew/control_collection_shard1_replica_n1/ shard1
   [junit4]   2> 2540672 INFO  (zkCallback-5329-thread-2-processing-n:127.0.0.1:62648_dl%2Few) [n:127.0.0.1:62648_dl%2Few    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 2540723 INFO  (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 2540732 INFO  (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1508
   [junit4]   2> 2540748 INFO  (qtp686307169-24869) [n:127.0.0.1:62648_dl%2Few    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 2540831 INFO  (zkCallback-5329-thread-2-processing-n:127.0.0.1:62648_dl%2Few) [n:127.0.0.1:62648_dl%2Few    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 2541122 INFO  (OverseerCollectionConfigSetProcessor-72064476004220932-127.0.0.1:62648_dl%2Few-n_0000000000) [n:127.0.0.1:62648_dl%2Few    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 2541749 INFO  (qtp686307169-24869) [n:127.0.0.1:62648_dl%2Few    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:62648_dl%252Few&wt=javabin&version=2} status=0 QTime=2639
   [junit4]   2> 2541754 INFO  (zkConnectionManagerCallback-5343-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2541756 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2541777 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:54034/solr ready
   [junit4]   2> 2541777 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 2541777 INFO  (SocketProxy-Acceptor-62648) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=62448,localport=62648], receiveBufferSize:130880
   [junit4]   2> 2541778 INFO  (SocketProxy-Acceptor-62648) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=50816,localport=36628], receiveBufferSize=139264
   [junit4]   2> 2541778 INFO  (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=2&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2541786 INFO  (OverseerThreadFactory-9898-thread-2-processing-n:127.0.0.1:62648_dl%2Few) [n:127.0.0.1:62648_dl%2Few    ] o.a.s.c.a.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 2541788 WARN  (OverseerThreadFactory-9898-thread-2-processing-n:127.0.0.1:62648_dl%2Few) [n:127.0.0.1:62648_dl%2Few    ] o.a.s.c.a.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
   [junit4]   2> 2542061 INFO  (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 2542063 INFO  (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=2&wt=javabin&version=2} status=0 QTime=284
   [junit4]   2> 2542398 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest_95A7A0DC3B52F81B-001/shard-1-001 of type TLOG
   [junit4]   2> 2542399 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T16:27:37-05:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 2542399 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 2542399 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 2542399 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.e.j.s.session Scavenging every 660000ms
   [junit4]   2> 2542400 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1483ff35{/dl/ew,null,AVAILABLE}
   [junit4]   2> 2542400 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@559d3cc4{HTTP/1.1,[http/1.1]}{127.0.0.1:46791}
   [junit4]   2> 2542400 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.e.j.s.Server Started @2545928ms
   [junit4]   2> 2542400 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest_95A7A0DC3B52F81B-001/tempDir-001/jetty1, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/dl/ew, hostPort=52399, coreRootDirectory=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest_95A7A0DC3B52F81B-001/shard-1-001/cores}
   [junit4]   2> 2542401 ERROR (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2542401 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 2542401 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2542401 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2542401 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-03-15T15:36:58.930Z
   [junit4]   2> 2542422 INFO  (zkConnectionManagerCallback-5345-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2542429 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 2542429 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest_95A7A0DC3B52F81B-001/shard-1-001/solr.xml
   [junit4]   2> 2542433 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 2542433 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 2542434 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@b81b1b4, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 2542449 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:54034/solr
   [junit4]   2> 2542451 INFO  (zkConnectionManagerCallback-5349-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2542459 INFO  (zkConnectionManagerCallback-5351-thread-1-processing-n:127.0.0.1:52399_dl%2Few) [n:127.0.0.1:52399_dl%2Few    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2542486 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:52399_dl%2Few    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2542492 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:52399_dl%2Few    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2542500 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:52399_dl%2Few    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52399_dl%2Few
   [junit4]   2> 2542508 INFO  (zkCallback-5329-thread-2-processing-n:127.0.0.1:62648_dl%2Few) [n:127.0.0.1:62648_dl%2Few    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2542509 INFO  (zkCallback-5334-thread-1-processing-n:127.0.0.1:62648_dl%2Few) [n:127.0.0.1:62648_dl%2Few    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2542513 INFO  (zkCallback-5342-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2542528 INFO  (zkCallback-5350-thread-1-processing-n:127.0.0.1:52399_dl%2Few) [n:127.0.0.1:52399_dl%2Few    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2542699 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:52399_dl%2Few    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b81b1b4
   [junit4]   2> 2542712 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:52399_dl%2Few    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b81b1b4
   [junit4]   2> 2542712 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:52399_dl%2Few    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b81b1b4
   [junit4]   2> 2542714 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:52399_dl%2Few    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest_95A7A0DC3B52F81B-001/shard-1-001/cores
   [junit4]   2> 2542773 INFO  (zkConnectionManagerCallback-5356-thread-1-processing-n:127.0.0.1:52399_dl%2Few) [n:127.0.0.1:52399_dl%2Few    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2542774 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:52399_dl%2Few    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2542776 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:52399_dl%2Few    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:54034/solr ready
   [junit4]   2> 2542858 INFO  (SocketProxy-Acceptor-52399) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=35601,localport=52399], receiveBufferSize:130880
   [junit4]   2> 2542858 INFO  (SocketProxy-Acceptor-52399) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=46791,localport=54110], receiveBufferSize=139264
   [junit4]   2> 2542860 INFO  (qtp1572325424-24928) [n:127.0.0.1:52399_dl%2Few    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:52399_dl%252Few&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2542881 INFO  (OverseerCollectionConfigSetProcessor-72064476004220932-127.0.0.1:62648_dl%2Few-n_0000000000) [n:127.0.0.1:62648_dl%2Few    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 2542883 INFO  (OverseerThreadFactory-9898-thread-3-processing-n:127.0.0.1:62648_dl%2Few) [n:127.0.0.1:62648_dl%2Few    ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:52399_dl%2Few for creating new replica
   [junit4]   2> 2542905 INFO  (SocketProxy-Acceptor-52399) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=52806,localport=52399], receiveBufferSize:130880
   [junit4]   2> 2542909 INFO  (SocketProxy-Acceptor-52399) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=46791,localport=34758], receiveBufferSize=139264
   [junit4]   2> 2542910 INFO  (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t21&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 2542910 INFO  (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 2543031 INFO  (zkCallback-5350-thread-1-processing-n:127.0.0.1:52399_dl%2Few) [n:127.0.0.1:52399_dl%2Few    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2543961 INFO  (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 2543978 INFO  (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.s.IndexSchema [collection1_shard1_replica_t21] Schema name=test
   [junit4]   2> 2544101 INFO  (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 2544127 INFO  (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_t21' using configuration from collection collection1, trusted=true
   [junit4]   2> 2544128 INFO  (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_t21' (registry 'solr.core.collection1.shard1.replica_t21') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b81b1b4
   [junit4]   2> 2544128 INFO  (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 2544128 INFO  (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.SolrCore [[collection1_shard1_replica_t21] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest_95A7A0DC3B52F81B-001/shard-1-001/cores/collection1_shard1_replica_t21], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest_95A7A0DC3B52F81B-001/shard-1-001/cores/collection1_shard1_replica_t21/data/]
   [junit4]   2> 2544130 INFO  (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=28, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=4.8076171875, floorSegmentMB=1.9384765625, forceMergeDeletesPctAllowed=22.775887864813843, segmentsPerTier=13.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.31703850642037046
   [junit4]   2> 2544132 WARN  (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 2544187 INFO  (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2544187 INFO  (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=2 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2544188 INFO  (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2544188 INFO  (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2544189 INFO  (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=22, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 2544189 INFO  (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.s.SolrIndexSearcher Opening [Searcher@1766ac70[collection1_shard1_replica_t21] main]
   [junit4]   2> 2544191 INFO  (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 2544192 INFO  (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2544193 INFO  (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 2544193 INFO  (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1595018545171791872
   [junit4]   2> 2544194 INFO  (searcherExecutor-9912-thread-1-processing-n:127.0.0.1:52399_dl%2Few x:collection1_shard1_replica_t21 s:shard1 c:collection1) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.SolrCore [collection1_shard1_replica_t21] Registered new searcher Searcher@1766ac70[collection1_shard1_replica_t21] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2544200 INFO  (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.ZkShardTerms Successful update of terms at /collections/collection1/terms/shard1 to Terms{values={core_node22=0}, version=0}
   [junit4]   2> 2544206 INFO  (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 2544206 INFO  (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 2544206 INFO  (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:52399/dl/ew/collection1_shard1_replica_t21/
   [junit4]   2> 2544206 INFO  (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 2544206 INFO  (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.SyncStrategy http://127.0.0.1:52399/dl/ew/collection1_shard1_replica_t21/ has no replicas
   [junit4]   2> 2544206 INFO  (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 2544207 INFO  (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.ZkController collection1_shard1_replica_t21 stopping background replication from leader
   [junit4]   2> 2544213 INFO  (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:52399/dl/ew/collection1_shard1_replica_t21/ shard1
   [junit4]   2> 2544320 INFO  (zkCallback-5350-thread-1-processing-n:127.0.0.1:52399_dl%2Few) [n:127.0.0.1:52399_dl%2Few    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2544371 INFO  (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 2544374 INFO  (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t21&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=1464
   [junit4]   2> 2544378 INFO  (qtp1572325424-24928) [n:127.0.0.1:52399_dl%2Few    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:52399_dl%252Few&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2} status=0 QTime=1518
   [junit4]   2> 2544475 INFO  (zkCallback-5350-thread-1-processing-n:127.0.0.1:52399_dl%2Few) [n:127.0.0.1:52399_dl%2Few    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2544554 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest_95A7A0DC3B52F81B-001/shard-2-001 of type TLOG
   [junit4]   2> 2544555 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T16:27:37-05:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 2544563 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 2544564 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 2544564 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.e.j.s.session Scavenging every 600000ms
   [junit4]   2> 2544582 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7c562958{/dl/ew,null,AVAILABLE}
   [junit4]   2> 2544583 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@4f5da30a{HTTP/1.1,[http/1.1]}{127.0.0.1:60058}
   [junit4]   2> 2544583 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.e.j.s.Server Started @2548111ms
   [junit4]   2> 2544583 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest_95A7A0DC3B52F81B-001/tempDir-001/jetty2, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/dl/ew, hostPort=64943, coreRootDirectory=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest_95A7A0DC3B52F81B-001/shard-2-001/cores}
   [junit4]   2> 2544583 ERROR (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2544584 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 2544584 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2544584 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2544584 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-03-15T15:37:01.113Z
   [junit4]   2> 2544587 INFO  (zkConnectionManagerCallback-5358-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2544589 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 2544589 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest_95A7A0DC3B52F81B-001/shard-2-001/solr.xml
   [junit4]   2> 2544591 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 2544592 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 2544593 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@b81b1b4, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 2544595 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:54034/solr
   [junit4]   2> 2544603 INFO  (zkConnectionManagerCallback-5362-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2544606 INFO  (zkConnectionManagerCallback-5364-thread-1-processing-n:127.0.0.1:64943_dl%2Few) [n:127.0.0.1:64943_dl%2Few    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2544625 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:64943_dl%2Few    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2544634 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:64943_dl%2Few    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2544638 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:64943_dl%2Few    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:64943_dl%2Few
   [junit4]   2> 2544640 INFO  (zkCallback-5334-thread-1-processing-n:127.0.0.1:62648_dl%2Few) [n:127.0.0.1:62648_dl%2Few    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2544640 INFO  (zkCallback-5342-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2544640 INFO  (zkCallback-5350-thread-1-processing-n:127.0.0.1:52399_dl%2Few) [n:127.0.0.1:52399_dl%2Few    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2544641 INFO  (zkCallback-5329-thread-2-processing-n:127.0.0.1:62648_dl%2Few) [n:127.0.0.1:62648_dl%2Few    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2544642 INFO  (zkCallback-5355-thread-1-processing-n:127.0.0.1:52399_dl%2Few) [n:127.0.0.1:52399_dl%2Few    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2544645 INFO  (zkCallback-5363-thread-1-processing-n:127.0.0.1:64943_dl%2Few) [n:127.0.0.1:64943_dl%2Few    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2544767 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:64943_dl%2Few    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b81b1b4
   [junit4]   2> 2544780 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:64943_dl%2Few    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b81b1b4
   [junit4]   2> 2544780 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:64943_dl%2Few    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b81b1b4
   [junit4]   2> 2544781 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:64943_dl%2Few    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest_95A7A0DC3B52F81B-001/shard-2-001/cores
   [junit4]   2> 2544789 INFO  (zkConnectionManagerCallback-5369-thread-1-processing-n:127.0.0.1:64943_dl%2Few) [n:127.0.0.1:64943_dl%2Few    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2544791 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:64943_dl%2Few    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 2544793 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:64943_dl%2Few    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:54034/solr ready
   [junit4]   2> 2544842 INFO  (qtp1572325424-24940) [n:127.0.0.1:52399_dl%2Few    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:64943_dl%252Few&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2544845 INFO  (OverseerCollectionConfigSetProcessor-72064476004220932-127.0.0.1:62648_dl%2Few-n_0000000000) [n:127.0.0.1:62648_dl%2Few    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 2544847 INFO  (OverseerThreadFactory-9898-thread-4-processing-n:127.0.0.1:62648_dl%2Few) [n:127.0.0.1:62648_dl%2Few    ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:64943_dl%2Few for creating new replica
   [junit4]   2> 2544852 INFO  (SocketProxy-Acceptor-64943) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=36438,localport=64943], receiveBufferSize:130880
   [junit4]   2> 2544853 INFO  (SocketProxy-Acceptor-64943) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=60058,localport=54002], receiveBufferSize=139264
   [junit4]   2> 2544853 INFO  (qtp1397148686-24971) [n:127.0.0.1:64943_dl%2Few    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t23&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 2544854 INFO  (qtp1397148686-24971) [n:127.0.0.1:64943_dl%2Few    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 2544960 INFO  (zkCallback-5350-thread-1-processing-n:127.0.0.1:52399_dl%2Few) [n:127.0.0.1:52399_dl%2Few    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 2544961 INFO  (zkCallback-5363-thread-1-processing-n:127.0.0.1:64943_dl%2Few) [n:127.0.0.1:64943_dl%2Few    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 2545874 INFO  (qtp1397148686-24971) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t23] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 2545884 INFO  (qtp1397148686-24971) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t23] o.a.s.s.IndexSchema [collection1_shard1_replica_t23] Schema name=test
   [junit4]   2> 2545965 INFO  (qtp1397148686-24971) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t23] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 2545985 INFO  (qtp1397148686-24971) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t23] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_t23' using configuration from collection collection1, trusted=true
   [junit4]   2> 2545985 INFO  (qtp1397148686-24971) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t23] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_t23' (registry 'solr.core.collection1.shard1.replica_t23') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b81b1b4
   [junit4]   2> 2545985 INFO  (qtp1397148686-24971) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t23] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 2545986 INFO  (qtp1397148686-24971) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t23] o.a.s.c.SolrCore [[collection1_shard1_replica_t23] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest_95A7A0DC3B52F81B-001/shard-2-001/cores/collection1_shard1_replica_t23], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest_95A7A0DC3B52F81B-001/shard-2-001/cores/collection1_shard1_replica_t23/data/]
   [junit4]   2> 2545987 INFO  (qtp1397148686-24971) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t23] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=28, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=4.8076171875, floorSegmentMB=1.9384765625, forceMergeDeletesPctAllowed=22.775887864813843, segmentsPerTier=13.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.31703850642037046
   [junit4]   2> 2545989 WARN  (qtp1397148686-24971) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t23] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 2546026 INFO  (qtp1397148686-24971) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t23] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2546026 INFO  (qtp1397148686-24971) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t23] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=2 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2546027 INFO  (qtp1397148686-24971) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t23] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2546027 INFO  (qtp1397148686-24971) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t23] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2546028 INFO  (qtp1397148686-24971) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t23] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=22, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 2546028 INFO  (qtp1397148686-24971) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t23] o.a.s.s.SolrIndexSearcher Opening [Searcher@78d81afa[collection1_shard1_replica_t23] main]
   [junit4]   2> 2546029 INFO  (qtp1397148686-24971) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t23] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 2546030 INFO  (qtp1397148686-24971) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t23] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2546030 INFO  (qtp1397148686-24971) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t23] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 2546031 INFO  (searcherExecutor-9923-thread-1-processing-n:127.0.0.1:64943_dl%2Few x:collection1_shard1_replica_t23 s:shard1 c:collection1) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t23] o.a.s.c.SolrCore [collection1_shard1_replica_t23] Registered new searcher Searcher@78d81afa[collection1_shard1_replica_t23] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2546031 INFO  (qtp1397148686-24971) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t23] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1595018547099074560
   [junit4]   2> 2546035 INFO  (qtp1397148686-24971) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t23] o.a.s.c.ZkShardTerms Successful update of terms at /collections/collection1/terms/shard1 to Terms{values={core_node24=0, core_node22=0}, version=1}
   [junit4]   2> 2546037 INFO  (qtp1397148686-24971) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t23] o.a.s.c.ZkController Core needs to recover:collection1_shard1_replica_t23
   [junit4]   2> 2546037 INFO  (updateExecutor-5359-thread-1-processing-n:127.0.0.1:64943_dl%2Few x:collection1_shard1_replica_t23 s:shard1 c:collection1) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_t23] o.a.s.u.DefaultSolrCoreState Running recovery
   [junit4]   2> 2546037 INFO  (recoveryExecutor-5360-thread-1-processing-n:127.0.0.1:64943_dl%2Few x:collection1_shard1_replica_t23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_t23] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
   [junit4]   2> 2546037 INFO  (recoveryExecutor-5360-thread-1-processing-n:127.0.0.1:64943_dl%2Few x:collection1_shard1_replica_t23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_t23] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
   [junit4]   2> 2546037 INFO  (recoveryExecutor-5360-thread-1-processing-n:127.0.0.1:64943_dl%2Few x:collection1_shard1_replica_t23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_t23] o.a.s.c.ZkController collection1_shard1_replica_t23 stopping background replication from leader
   [junit4]   2> 2546037 INFO  (qtp1397148686-24971) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1  x:collection1_shard1_replica_t23] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t23&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=1184
   [junit4]   2> 2546042 INFO  (SocketProxy-Acceptor-52399) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=53634,localport=52399], receiveBufferSize:130880
   [junit4]   2> 2546043 INFO  (SocketProxy-Acceptor-52399) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=46791,localport=48703], receiveBufferSize=139264
   [junit4]   2> 2546043 INFO  (qtp1572325424-24932) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1 r:core_node22 x:collection1_shard1_replica_t21] o.a.s.c.S.Request [collection1_shard1_replica_t21]  webapp=/dl/ew path=/admin/ping params={wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 2546043 INFO  (qtp1572325424-24932) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1 r:core_node22 x:collection1_shard1_replica_t21] o.a.s.c.S.Request [collection1_shard1_replica_t21]  webapp=/dl/ew path=/admin/ping params={wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 2546044 INFO  (qtp1572325424-24940) [n:127.0.0.1:52399_dl%2Few    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:64943_dl%252Few&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2} status=0 QTime=1202
   [junit4]   2> 2546044 INFO  (recoveryExecutor-5360-thread-1-processing-n:127.0.0.1:64943_dl%2Few x:collection1_shard1_replica_t23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_t23] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1_shard1_replica_t23]
   [junit4]   2> 2546044 INFO  (recoveryExecutor-5360-thread-1-processing-n:127.0.0.1:64943_dl%2Few x:collection1_shard1_replica_t23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_t23] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
   [junit4]   2> 2546044 INFO  (recoveryExecutor-5360-thread-1-processing-n:127.0.0.1:64943_dl%2Few x:collection1_shard1_replica_t23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_t23] o.a.s.c.RecoveryStrategy Publishing state of core [collection1_shard1_replica_t23] as recovering, leader is [http://127.0.0.1:52399/dl/ew/collection1_shard1_replica_t21/] and I am [http://127.0.0.1:64943/dl/ew/collection1_shard1_replica_t23/]
   [junit4]   2> 2546046 INFO  (recoveryExecutor-5360-thread-1-processing-n:127.0.0.1:64943_dl%2Few x:collection1_shard1_replica_t23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_t23] o.a.s.c.ZkShardTerms Successful update of terms at /collections/collection1/terms/shard1 to Terms{values={core_node24_recovering=0, core_node24=0, core_node22=0}, version=2}
   [junit4]   2> 2546048 INFO  (recoveryExecutor-5360-thread-1-processing-n:127.0.0.1:64943_dl%2Few x:collection1_shard1_replica_t23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_t23] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:52399/dl/ew]; [WaitForState: action=PREPRECOVERY&core=collection1_shard1_replica_t21&nodeName=127.0.0.1:64943_dl%252Few&coreNodeName=core_node24&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
   [junit4]   2> 2546048 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.SolrTestCaseJ4 ###Starting test
   [junit4]   2> 2546048 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [    ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: collection1 failOnTimeout:true timeout (sec):330
   [junit4]   1> -
   [junit4]   2> 2546049 INFO  (SocketProxy-Acceptor-52399) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=55334,localport=52399], receiveBufferSize:130880
   [junit4]   2> 2546049 INFO  (SocketProxy-Acceptor-52399) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=46791,localport=62663], receiveBufferSize=139264
   [junit4]   1> replica:core_node22 rstate:active live:true
   [junit4]   1> replica:core_node24 rstate:down live:true
   [junit4]   2> 2546050 INFO  (qtp1572325424-24940) [n:127.0.0.1:52399_dl%2Few    ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node24, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true, maxTime: 183 s
   [junit4]   2> 2546050 INFO  (qtp1572325424-24940) [n:127.0.0.1:52399_dl%2Few    ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1_shard1_replica_t21, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:64943_dl%2Few, coreNodeName=core_node24, onlyIfActiveCheckResult=false, nodeProps: core_node24:{"core":"collection1_shard1_replica_t23","base_url":"http://127.0.0.1:64943/dl/ew","node_name":"127.0.0.1:64943_dl%2Few","state":"down","type":"TLOG"}
   [junit4]   2> 2546149 INFO  (zkCallback-5363-thread-1-processing-n:127.0.0.1:64943_dl%2Few) [n:127.0.0.1:64943_dl%2Few    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 2546149 INFO  (zkCallback-5350-thread-1-processing-n:127.0.0.1:52399_dl%2Few) [n:127.0.0.1:52399_dl%2Few    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 2546848 INFO  (OverseerCollectionConfigSetProcessor-72064476004220932-127.0.0.1:62648_dl%2Few-n_0000000000) [n:127.0.0.1:62648_dl%2Few    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000006 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   1> -
   [junit4]   1> replica:core_node22 rstate:active live:true
   [junit4]   1> replica:core_node24 rstate:recovering live:true
   [junit4]   2> 2547051 INFO  (qtp1572325424-24940) [n:127.0.0.1:52399_dl%2Few    ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1_shard1_replica_t21, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:64943_dl%2Few, coreNodeName=core_node24, onlyIfActiveCheckResult=false, nodeProps: core_node24:{"core":"collection1_shard1_replica_t23","base_url":"http://127.0.0.1:64943/dl/ew","node_name":"127.0.0.1:64943_dl%2Few","state":"recovering","type":"TLOG"}
   [junit4]   2> 2547051 INFO  (qtp1572325424-24940) [n:127.0.0.1:52399_dl%2Few    ] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node24, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
   [junit4]   2> 2547051 INFO  (qtp1572325424-24940) [n:127.0.0.1:52399_dl%2Few    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:64943_dl%252Few&onlyIfLeaderActive=true&core=collection1_shard1_replica_t21&coreNodeName=core_node24&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=1002
   [junit4]   2> 2547553 INFO  (recoveryExecutor-5360-thread-1-processing-n:127.0.0.1:64943_dl%2Few x:collection1_shard1_replica_t23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_t23] o.a.s.c.RecoveryStrategy Starting Replication Recovery.
   [junit4]   2> 2547553 INFO  (recoveryExecutor-5360-thread-1-processing-n:127.0.0.1:64943_dl%2Few x:collection1_shard1_replica_t23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_t23] o.a.s.c.RecoveryStrategy Attempting to replicate from [http://127.0.0.1:52399/dl/ew/collection1_shard1_replica_t21/].
   [junit4]   2> 2547554 INFO  (SocketProxy-Acceptor-52399) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=39896,localport=52399], receiveBufferSize:130880
   [junit4]   2> 2547555 INFO  (SocketProxy-Acceptor-52399) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=46791,localport=39551], receiveBufferSize=139264
   [junit4]   2> 2547555 INFO  (qtp1572325424-24940) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1 r:core_node22 x:collection1_shard1_replica_t21] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1595018548697104384,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 2547555 INFO  (qtp1572325424-24940) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1 r:core_node22 x:collection1_shard1_replica_t21] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 2547555 INFO  (qtp1572325424-24940) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1 r:core_node22 x:collection1_shard1_replica_t21] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 2547555 INFO  (qtp1572325424-24940) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1 r:core_node22 x:collection1_shard1_replica_t21] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_t21]  webapp=/dl/ew path=/update params={waitSearcher=true&openSearcher=false&commit=true&softCommit=false&commit_end_point=true&wt=javabin&version=2}{commit=} 0 0
   [junit4]   2> 2547557 INFO  (SocketProxy-Acceptor-52399) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=56444,localport=52399], receiveBufferSize:130880
   [junit4]   2> 2547557 INFO  (SocketProxy-Acceptor-52399) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=46791,localport=53659], receiveBufferSize=139264
   [junit4]   2> 2547558 INFO  (qtp1572325424-24940) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1 r:core_node22 x:collection1_shard1_replica_t21] o.a.s.c.S.Request [collection1_shard1_replica_t21]  webapp=/dl/ew path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
   [junit4]   2> 2547558 INFO  (recoveryExecutor-5360-thread-1-processing-n:127.0.0.1:64943_dl%2Few x:collection1_shard1_replica_t23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_t23] o.a.s.h.IndexFetcher Master's generation: 1
   [junit4]   2> 2547558 INFO  (recoveryExecutor-5360-thread-1-processing-n:127.0.0.1:64943_dl%2Few x:collection1_shard1_replica_t23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_t23] o.a.s.h.IndexFetcher Master's version: 0
   [junit4]   2> 2547558 INFO  (recoveryExecutor-5360-thread-1-processing-n:127.0.0.1:64943_dl%2Few x:collection1_shard1_replica_t23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_t23] o.a.s.h.IndexFetcher Slave's generation: 1
   [junit4]   2> 2547558 INFO  (recoveryExecutor-5360-thread-1-processing-n:127.0.0.1:64943_dl%2Few x:collection1_shard1_replica_t23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_t23] o.a.s.h.IndexFetcher Slave's version: 0
   [junit4]   2> 2547559 INFO  (recoveryExecutor-5360-thread-1-processing-n:127.0.0.1:64943_dl%2Few x:collection1_shard1_replica_t23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_t23] o.a.s.h.IndexFetcher New index in Master. Deleting mine...
   [junit4]   2> 2547559 INFO  (recoveryExecutor-5360-thread-1-processing-n:127.0.0.1:64943_dl%2Few x:collection1_shard1_replica_t23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_t23] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 2547559 INFO  (recoveryExecutor-5360-thread-1-processing-n:127.0.0.1:64943_dl%2Few x:collection1_shard1_replica_t23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_t23] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@4b5e13f8 commitCommandVersion:0
   [junit4]   2> 2547559 INFO  (recoveryExecutor-5360-thread-1-processing-n:127.0.0.1:64943_dl%2Few x:collection1_shard1_replica_t23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_t23] o.a.s.s.SolrIndexSearcher Opening [Searcher@6fb8fcf6[collection1_shard1_replica_t23] main]
   [junit4]   2> 2547559 INFO  (recoveryExecutor-5360-thread-1-processing-n:127.0.0.1:64943_dl%2Few x:collection1_shard1_replica_t23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_t23] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 2547561 INFO  (searcherExecutor-9923-thread-1-processing-n:127.0.0.1:64943_dl%2Few x:collection1_shard1_replica_t23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_t23] o.a.s.c.SolrCore [collection1_shard1_replica_t23] Registered new searcher Searcher@6fb8fcf6[collection1_shard1_replica_t23] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2547561 INFO  (recoveryExecutor-5360-thread-1-processing-n:127.0.0.1:64943_dl%2Few x:collection1_shard1_replica_t23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_t23] o.a.s.c.RecoveryStrategy Replication Recovery was successful.
   [junit4]   2> 2547562 INFO  (recoveryExecutor-5360-thread-1-processing-n:127.0.0.1:64943_dl%2Few x:collection1_shard1_replica_t23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_t23] o.a.s.c.RecoveryStrategy Registering as Active after recovery.
   [junit4]   2> 2547562 INFO  (recoveryExecutor-5360-thread-1-processing-n:127.0.0.1:64943_dl%2Few x:collection1_

[...truncated too long message...]

est, TestReplicationHandlerBackup, TestCloudManagedSchema, DistribJoinFromCollectionTest, AliasIntegrationTest, TestCollationFieldDocValues, TestLegacyTerms, SolrXmlInZkTest, TestHighlightDedupGrouping, TestDelegationWithHadoopAuth, RAMDirectoryFactoryTest, HdfsWriteToMultipleCollectionsTest, TestUseDocValuesAsStored2, FacetPivotSmallTest, TestDFISimilarityFactory, ZkShardTermsTest, ConvertedLegacyTest, CollectionsAPISolrJTest, TestMiniSolrCloudClusterSSL, TestJsonFacets, TestCloudConsistency, TestManagedResourceStorage, TestRequestForwarding, HttpSolrCallGetCoreTest, TriggerIntegrationTest, HttpPartitionOnCommitTest, TestIndexSearcher, TestXmlQParser, CoreSorterTest, PolyFieldTest, TestSweetSpotSimilarityFactory, TestTlogReplica, MoveReplicaHDFSTest, RuleEngineTest, FastVectorHighlighterTest, TestConfigsApi, SolrMetricReporterTest, CurrencyFieldTypeTest, DirectUpdateHandlerOptimizeTest, SpatialRPTFieldTypeTest, TestSolrCloudWithDelegationTokens, TestCollationField, JvmMetricsTest, SpatialHeatmapFacetsTest, TestClusterStateMutator, TestSchemaManager, TestSizeLimitedDistributedMap, TestGraphMLResponseWriter, LeaderElectionTest, TestFieldSortValues, DistributedFacetPivotSmallAdvancedTest, TestExactSharedStatsCache, TestLRUStatsCache, TestPseudoReturnFields, TestFaceting, TestFieldCacheReopen, TestSerializedLuceneMatchVersion, DirectSolrSpellCheckerTest, SortSpecParsingTest, LIROnShardRestartTest, SchemaVersionSpecificBehaviorTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, LukeRequestHandlerTest, HdfsSyncSliceTest, TestReRankQParserPlugin, TestRangeQuery, TestScoreJoinQPNoScore, LeaderFailureAfterFreshStartTest, SimpleCollectionCreateDeleteTest, TestLuceneIndexBackCompat, TestJettySolrRunner, SchemaApiFailureTest, BlockJoinFacetDistribTest, SolrPluginUtilsTest, TermVectorComponentDistributedTest, DistributedMLTComponentTest, TestReloadAndDeleteDocs, VMParamsZkACLAndCredentialsProvidersTest, UpdateLogTest, SolrIndexSplitterTest, NumericFieldsTest, NumberUtilsTest, AlternateDirectoryTest, TestSurroundQueryParser, SystemLogListenerTest, TestLegacyFieldReuse, StatelessScriptUpdateProcessorFactoryTest, TestConfigOverlay, TestLFUCache, MetricUtilsTest, AtomicUpdatesTest, AddSchemaFieldsUpdateProcessorFactoryTest, AsyncCallRequestStatusResponseTest, TestDocTermOrds, TestDistribDocBasedVersion, ReplicationFactorTest, TestCharFilters, CustomHighlightComponentTest, TestDownShardTolerantSearch, TestTolerantSearch, ClassificationUpdateProcessorIntegrationTest, SpellingQueryConverterTest, TestCSVResponseWriter, TestNumericTokenStream, TestBM25SimilarityFactory, HighlighterTest, TestSQLHandlerNonCloud, TestCryptoKeys, AnalyticsQueryTest, DistributedQueryElevationComponentTest, ZkNodePropsTest, TestFieldTypeCollectionResource, TestBinaryField, TestQueryUtils, HdfsChaosMonkeyNothingIsSafeTest, AnalysisErrorHandlingTest, TestRandomCollapseQParserPlugin, CursorMarkTest, NodeLostTriggerTest, TestPerFieldSimilarityWithDefaultOverride, TestRandomRequestDistribution, ConfigSetsAPITest, TestInPlaceUpdatesDistrib, SynonymTokenizerTest, CloneFieldUpdateProcessorFactoryTest, DistributedQueryComponentOptimizationTest, BasicZkTest, ResourceLoaderTest, TestSubQueryTransformer, TestCoreBackup, TestMaxScoreQueryParser, TemplateUpdateProcessorTest, UUIDUpdateProcessorFallbackTest, TestExportWriter, TestPivotHelperCode, TestMinMaxOnMultiValuedField, TestSolrFieldCacheBean, TestLeaderElectionWithEmptyReplica, BigEndianAscendingWordDeserializerTest, TestZkChroot, UniqFieldsUpdateProcessorFactoryTest, TestConfigSetProperties, TestStressUserVersions, PingRequestHandlerTest, MBeansHandlerTest, TestTolerantUpdateProcessorRandomCloud, QueryEqualityTest, ConcurrentDeleteAndCreateCollectionTest, DistributedVersionInfoTest, TestJmxIntegration, BaseCdcrDistributedZkTest, SolrShardReporterTest, TestOverriddenPrefixQueryForCustomFieldType, CdcrReplicationHandlerTest, TestPostingsSolrHighlighter, CollectionReloadTest, TermsComponentTest, TestReloadDeadlock, FileUtilsTest, TestComplexPhraseQParserPlugin, QueryElevationComponentTest, TestMacroExpander, CachingDirectoryFactoryTest, TestLargeCluster, JSONWriterTest, TestImplicitCoreProperties, TestFileDictionaryLookup, BasicDistributedZkTest, SimpleFacetsTest, SolrCoreTest, TestGroupingSearch, SolrCmdDistributorTest, PeerSyncTest, DirectUpdateHandlerTest, TestBadConfig, SimplePostToolTest, AnalysisAfterCoreReloadTest, SignatureUpdateProcessorFactoryTest, CoreAdminHandlerTest, SolrRequestParserTest, DocValuesTest, SuggesterTSTTest, SuggesterTest, TestStressLucene, TestTrie, SpatialFilterTest, SuggesterWFSTTest, TestCSVLoader, WordBreakSolrSpellCheckerTest, FieldMutatingUpdateProcessorTest, TestAtomicUpdateErrorCases, TestWordDelimiterFilterFactory, DefaultValueUpdateProcessorTest, TestRemoteStreaming, DocValuesMultiTest, TestSolrDeletionPolicy1, SolrInfoBeanTest, XsltUpdateRequestHandlerTest, CacheHeaderTest, DocumentAnalysisRequestHandlerTest, TestQueryTypes, TestOmitPositions, XmlUpdateRequestHandlerTest, RequestHandlersTest, DocumentBuilderTest, PathHierarchyTokenizerFactoryTest, TestIndexingPerformance, MoreLikeThisHandlerTest, RequiredFieldsTest, TestSolrQueryParser, IndexSchemaRuntimeFieldTest, UpdateRequestProcessorFactoryTest, JsonLoaderTest, BinaryUpdateRequestHandlerTest, TestComponentsName, SearchHandlerTest, SOLR749Test, TestQuerySenderListener, UpdateParamsTest, CopyFieldTest, ResponseLogComponentTest, SolrIndexConfigTest, BadComponentTest, TestMergePolicyConfig, MultiTermTest, SampleTest, TestDocSet, MinimalSchemaTest, TestElisionMultitermQuery, OutputWriterTest, TestSolrCoreProperties, DirectSolrConnectionTest, NotRequiredUniqueKeyTest, TestPhraseSuggestions, SpellPossibilityIteratorTest, TestCodecSupport, TestXIncludeConfig, TestDFRSimilarityFactory, TestPerFieldSimilarity, TestLMDirichletSimilarityFactory, TestLMJelinekMercerSimilarityFactory, TestIBSimilarityFactory, TestFastWriter, TestFastOutputStream, OpenExchangeRatesOrgProviderTest, URLClassifyProcessorTest, PluginInfoTest, DateMathParserTest, PreAnalyzedFieldTest, TestSystemIdResolver, PrimUtilsTest, TestSuggestSpellingConverter, DOMUtilTest, TestSolrJ, TestLRUCache, TestUtils, SliceStateTest, TestRTGBase, CursorPagingTest, SolrTestCaseJ4Test, TestCrossCoreJoin, TestSimpleTrackingShardHandler, AssignBackwardCompatibilityTest, ConnectionManagerTest, CreateCollectionCleanupTest, DeleteNodeTest, DeleteStatusTest, DistributedQueueTest, DocValuesNotIndexedTest, ForceLeaderTest, HealthCheckHandlerTest, HttpPartitionTest, LeaderElectionContextKeyTest, MissingSegmentRecoveryTest, MultiThreadedOCPTest, OutOfBoxZkACLAndCredentialsProvidersTest, OverriddenZkACLAndCredentialsProvidersTest, OverseerCollectionConfigSetProcessorTest, OverseerModifyCollectionTest, RecoveryAfterSoftCommitTest]
   [junit4] Completed [580/785 (1!)] on J0 in 642.29s, 1 test, 1 failure <<< FAILURES!

[...truncated 36779 lines...]
-ecj-javadoc-lint-src:
    [mkdir] Created dir: /var/tmp/ecj343962066
 [ecj-lint] Compiling 827 source files to /var/tmp/ecj343962066
 [ecj-lint] ----------
 [ecj-lint] 1. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/lucene/core/src/java/org/apache/lucene/codecs/CodecUtil.java (at line 523)
 [ecj-lint] 	throw new CorruptIndexException("misplaced codec footer (file truncated?): length=" + in.length() + " but footerLength==" + footerLength(), input);
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'in' is not closed at this location
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 2. ERROR in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java (at line 738)
 [ecj-lint] 	} finally {
 [ecj-lint]           throw t;
 [ecj-lint]         }
 [ecj-lint] 	          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] finally block does not complete normally
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 3. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java (at line 841)
 [ecj-lint] 	SegmentReader newReader = new SegmentReader(info, reader, liveDocs, info.info.maxDoc() - delCount);
 [ecj-lint] 	              ^^^^^^^^^
 [ecj-lint] Resource leak: 'newReader' is never closed
 [ecj-lint] ----------
 [ecj-lint] 3 problems (1 error, 2 warnings)

BUILD FAILED
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/build.xml:618: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/build.xml:101: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/lucene/build.xml:203: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/lucene/common-build.xml:2082: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/lucene/common-build.xml:2121: Compile failed; see the compiler error output for details.

Total time: 92 minutes 32 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2

[JENKINS] Lucene-Solr-master-Solaris (64bit/jdk1.8.0) - Build # 1736 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1736/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseG1GC

5 tests failed.
FAILED:  org.apache.solr.cloud.autoscaling.ScheduledTriggerTest.testTrigger

Error Message:
Did not expect the listener to fire on first run!

Stack Trace:
java.lang.AssertionError: Did not expect the listener to fire on first run!
	at __randomizedtesting.SeedInfo.seed([16B355D9DBFA6A65:7578635B42351948]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.autoscaling.ScheduledTriggerTest.lambda$new$0(ScheduledTriggerTest.java:48)
	at org.apache.solr.cloud.autoscaling.ScheduledTrigger.run(ScheduledTrigger.java:191)
	at org.apache.solr.cloud.autoscaling.ScheduledTriggerTest.scheduledTriggerTest(ScheduledTriggerTest.java:102)
	at org.apache.solr.cloud.autoscaling.ScheduledTriggerTest.testTrigger(ScheduledTriggerTest.java:75)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.core.TestLazyCores

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
	at __randomizedtesting.SeedInfo.seed([16B355D9DBFA6A65]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.core.TestLazyCores

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
	at __randomizedtesting.SeedInfo.seed([16B355D9DBFA6A65]:0)


FAILED:  org.apache.solr.core.TestLazyCores.testLazySearch

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
	at __randomizedtesting.SeedInfo.seed([16B355D9DBFA6A65]:0)


FAILED:  org.apache.solr.core.TestLazyCores.testLazySearch

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
	at __randomizedtesting.SeedInfo.seed([16B355D9DBFA6A65]:0)




Build Log:
[...truncated 1841 lines...]
   [junit4] JVM J1: stdout was not empty, see: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/lucene/build/core/test/temp/junit4-J1-20180316_083419_2518528479762677445149.sysout
   [junit4] >>> JVM J1 emitted unexpected output (verbatim) ----
   [junit4] codec: FastDecompressionCompressingStoredFields, pf: LuceneFixedGap, dvf: Memory
   [junit4] <<< JVM J1: EOF ----

[...truncated 11644 lines...]
   [junit4] Suite: org.apache.solr.cloud.autoscaling.ScheduledTriggerTest
   [junit4]   2> 4058877 INFO  (SUITE-ScheduledTriggerTest-seed#[16B355D9DBFA6A65]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.ScheduledTriggerTest_16B355D9DBFA6A65-001/init-core-data-001
   [junit4]   2> 4058877 INFO  (SUITE-ScheduledTriggerTest-seed#[16B355D9DBFA6A65]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 4058878 INFO  (SUITE-ScheduledTriggerTest-seed#[16B355D9DBFA6A65]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 4058879 INFO  (SUITE-ScheduledTriggerTest-seed#[16B355D9DBFA6A65]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 1 servers in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.ScheduledTriggerTest_16B355D9DBFA6A65-001/tempDir-001
   [junit4]   2> 4058879 INFO  (SUITE-ScheduledTriggerTest-seed#[16B355D9DBFA6A65]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 4058879 INFO  (Thread-10832) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 4058879 INFO  (Thread-10832) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 4058881 ERROR (Thread-10832) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 4058979 INFO  (SUITE-ScheduledTriggerTest-seed#[16B355D9DBFA6A65]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:50783
   [junit4]   2> 4058987 INFO  (zkConnectionManagerCallback-11034-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4058993 INFO  (jetty-launcher-11031-thread-1) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T00:27:37+03:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 4058994 INFO  (jetty-launcher-11031-thread-1) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 4058994 INFO  (jetty-launcher-11031-thread-1) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 4058994 INFO  (jetty-launcher-11031-thread-1) [    ] o.e.j.s.session Scavenging every 660000ms
   [junit4]   2> 4058994 INFO  (jetty-launcher-11031-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@39bb82cd{/solr,null,AVAILABLE}
   [junit4]   2> 4058994 INFO  (jetty-launcher-11031-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@1752b9f7{HTTP/1.1,[http/1.1]}{127.0.0.1:43318}
   [junit4]   2> 4058994 INFO  (jetty-launcher-11031-thread-1) [    ] o.e.j.s.Server Started @4063734ms
   [junit4]   2> 4058995 INFO  (jetty-launcher-11031-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=43318}
   [junit4]   2> 4058995 ERROR (jetty-launcher-11031-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 4058995 INFO  (jetty-launcher-11031-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solrâ„¢ version 8.0.0
   [junit4]   2> 4058995 INFO  (jetty-launcher-11031-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 4058995 INFO  (jetty-launcher-11031-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 4058995 INFO  (jetty-launcher-11031-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-03-16T09:59:26.227Z
   [junit4]   2> 4058997 INFO  (zkConnectionManagerCallback-11036-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4058997 INFO  (jetty-launcher-11031-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 4059004 INFO  (jetty-launcher-11031-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:50783/solr
   [junit4]   2> 4059005 INFO  (zkConnectionManagerCallback-11040-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4059008 INFO  (zkConnectionManagerCallback-11042-thread-1-processing-n:127.0.0.1:43318_solr) [n:127.0.0.1:43318_solr    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4059137 INFO  (jetty-launcher-11031-thread-1) [n:127.0.0.1:43318_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 4059138 INFO  (jetty-launcher-11031-thread-1) [n:127.0.0.1:43318_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:43318_solr
   [junit4]   2> 4059139 INFO  (jetty-launcher-11031-thread-1) [n:127.0.0.1:43318_solr    ] o.a.s.c.Overseer Overseer (id=72068813957758979-127.0.0.1:43318_solr-n_0000000000) starting
   [junit4]   2> 4059149 INFO  (zkConnectionManagerCallback-11047-thread-1-processing-n:127.0.0.1:43318_solr) [n:127.0.0.1:43318_solr    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4059152 INFO  (jetty-launcher-11031-thread-1) [n:127.0.0.1:43318_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:50783/solr ready
   [junit4]   2> 4059153 INFO  (jetty-launcher-11031-thread-1) [n:127.0.0.1:43318_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43318_solr
   [junit4]   2> 4059153 DEBUG (OverseerAutoScalingTriggerThread-72068813957758979-127.0.0.1:43318_solr-n_0000000000) [n:127.0.0.1:43318_solr    ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
   [junit4]   2> 4059154 DEBUG (OverseerAutoScalingTriggerThread-72068813957758979-127.0.0.1:43318_solr-n_0000000000) [n:127.0.0.1:43318_solr    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
   [junit4]   2> 4059155 INFO  (OverseerStateUpdate-72068813957758979-127.0.0.1:43318_solr-n_0000000000) [n:127.0.0.1:43318_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 4059155 DEBUG (OverseerAutoScalingTriggerThread-72068813957758979-127.0.0.1:43318_solr-n_0000000000) [n:127.0.0.1:43318_solr    ] o.a.s.c.a.NodeLostTrigger Initial livenodes: []
   [junit4]   2> 4059155 DEBUG (OverseerAutoScalingTriggerThread-72068813957758979-127.0.0.1:43318_solr-n_0000000000) [n:127.0.0.1:43318_solr    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion -1
   [junit4]   2> 4059155 DEBUG (OverseerAutoScalingTriggerThread-72068813957758979-127.0.0.1:43318_solr-n_0000000000) [n:127.0.0.1:43318_solr    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
   [junit4]   2> 4059156 INFO  (zkCallback-11046-thread-1-processing-n:127.0.0.1:43318_solr) [n:127.0.0.1:43318_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 4059160 DEBUG (OverseerAutoScalingTriggerThread-72068813957758979-127.0.0.1:43318_solr-n_0000000000) [n:127.0.0.1:43318_solr    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 4059161 DEBUG (OverseerAutoScalingTriggerThread-72068813957758979-127.0.0.1:43318_solr-n_0000000000) [n:127.0.0.1:43318_solr    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
   [junit4]   2> 4059161 DEBUG (ScheduledTrigger-14251-thread-1) [n:127.0.0.1:43318_solr    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
   [junit4]   2> 4059358 INFO  (jetty-launcher-11031-thread-1) [n:127.0.0.1:43318_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43318.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14887272
   [junit4]   2> 4059364 INFO  (jetty-launcher-11031-thread-1) [n:127.0.0.1:43318_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43318.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14887272
   [junit4]   2> 4059364 INFO  (jetty-launcher-11031-thread-1) [n:127.0.0.1:43318_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43318.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14887272
   [junit4]   2> 4059365 INFO  (jetty-launcher-11031-thread-1) [n:127.0.0.1:43318_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.ScheduledTriggerTest_16B355D9DBFA6A65-001/tempDir-001/node1/.
   [junit4]   2> 4059380 INFO  (zkConnectionManagerCallback-11049-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4059384 INFO  (zkConnectionManagerCallback-11053-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4059387 INFO  (SUITE-ScheduledTriggerTest-seed#[16B355D9DBFA6A65]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 4059388 INFO  (SUITE-ScheduledTriggerTest-seed#[16B355D9DBFA6A65]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:50783/solr ready
   [junit4]   2> 4059486 INFO  (TEST-ScheduledTriggerTest.testTrigger-seed#[16B355D9DBFA6A65]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testTrigger
   [junit4]   2> 4060166 DEBUG (ScheduledTrigger-14251-thread-3) [n:127.0.0.1:43318_solr    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
   [junit4]   2> 4061168 DEBUG (ScheduledTrigger-14251-thread-2) [n:127.0.0.1:43318_solr    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
   [junit4]   2> 4062168 DEBUG (ScheduledTrigger-14251-thread-2) [n:127.0.0.1:43318_solr    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
   [junit4]   2> 4062489 DEBUG (TEST-ScheduledTriggerTest.testTrigger-seed#[16B355D9DBFA6A65]) [    ] o.a.s.c.a.ScheduledTrigger ScheduledTrigger sched1 firing registered processor for scheduled time 2018-03-16T09:59:29.718Z, now=2018-03-16T09:59:32.289Z
   [junit4]   2> 4063172 DEBUG (ScheduledTrigger-14251-thread-2) [n:127.0.0.1:43318_solr    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
   [junit4]   2> 4064174 DEBUG (ScheduledTrigger-14251-thread-4) [n:127.0.0.1:43318_solr    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
   [junit4]   2> 4065182 DEBUG (ScheduledTrigger-14251-thread-2) [n:127.0.0.1:43318_solr    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
   [junit4]   2> 4065489 DEBUG (TEST-ScheduledTriggerTest.testTrigger-seed#[16B355D9DBFA6A65]) [    ] o.a.s.c.a.ScheduledTrigger ScheduledTrigger sched1 firing registered processor for scheduled time 2018-03-16T09:59:32.718Z, now=2018-03-16T09:59:35.290Z
   [junit4]   2> 4066183 DEBUG (ScheduledTrigger-14251-thread-2) [n:127.0.0.1:43318_solr    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
   [junit4]   2> 4067188 DEBUG (ScheduledTrigger-14251-thread-2) [n:127.0.0.1:43318_solr    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
   [junit4]   2> 4068188 DEBUG (ScheduledTrigger-14251-thread-2) [n:127.0.0.1:43318_solr    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
   [junit4]   2> 4068490 DEBUG (TEST-ScheduledTriggerTest.testTrigger-seed#[16B355D9DBFA6A65]) [    ] o.a.s.c.a.ScheduledTrigger ScheduledTrigger sched1 firing registered processor for scheduled time 2018-03-16T09:59:35.718Z, now=2018-03-16T09:59:38.291Z
   [junit4]   2> 4068492 DEBUG (TEST-ScheduledTriggerTest.testTrigger-seed#[16B355D9DBFA6A65]) [    ] o.a.s.c.a.ScheduledTrigger ScheduledTrigger sched1 firing registered processor for scheduled time 2018-03-16T09:59:38Z, now=2018-03-16T09:59:38.293Z
   [junit4]   2> 4068492 INFO  (TEST-ScheduledTriggerTest.testTrigger-seed#[16B355D9DBFA6A65]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testTrigger
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ScheduledTriggerTest -Dtests.method=testTrigger -Dtests.seed=16B355D9DBFA6A65 -Dtests.slow=true -Dtests.locale=id-ID -Dtests.timezone=America/Lima -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE 9.11s J1 | ScheduledTriggerTest.testTrigger <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: Did not expect the listener to fire on first run!
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([16B355D9DBFA6A65:7578635B42351948]:0)
   [junit4]    > 	at org.apache.solr.cloud.autoscaling.ScheduledTriggerTest.lambda$new$0(ScheduledTriggerTest.java:48)
   [junit4]    > 	at org.apache.solr.cloud.autoscaling.ScheduledTrigger.run(ScheduledTrigger.java:191)
   [junit4]    > 	at org.apache.solr.cloud.autoscaling.ScheduledTriggerTest.scheduledTriggerTest(ScheduledTriggerTest.java:102)
   [junit4]    > 	at org.apache.solr.cloud.autoscaling.ScheduledTriggerTest.testTrigger(ScheduledTriggerTest.java:75)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 4068662 INFO  (TEST-ScheduledTriggerTest.testIgnoredEvent-seed#[16B355D9DBFA6A65]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testIgnoredEvent
   [junit4]   2> 4068663 WARN  (TEST-ScheduledTriggerTest.testIgnoredEvent-seed#[16B355D9DBFA6A65]) [    ] o.a.s.c.a.ScheduledTrigger ScheduledTrigger was not able to run event at scheduled time: 2018-03-15T09:59:35.894Z. Now: 2018-03-16T09:59:38.465Z
   [junit4]   2> 4068664 INFO  (TEST-ScheduledTriggerTest.testIgnoredEvent-seed#[16B355D9DBFA6A65]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testIgnoredEvent
   [junit4]   2> 4068667 INFO  (jetty-closer-11032-thread-1) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@1752b9f7{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 4068670 INFO  (jetty-closer-11032-thread-1) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1052262676
   [junit4]   2> 4068671 INFO  (jetty-closer-11032-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 4068671 INFO  (jetty-closer-11032-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@3c0a4450: rootName = solr_43318, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@612cb6f
   [junit4]   2> 4068676 INFO  (jetty-closer-11032-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 4068676 INFO  (jetty-closer-11032-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@106784e1: rootName = solr_43318, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@686a409f
   [junit4]   2> 4068681 INFO  (jetty-closer-11032-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 4068681 INFO  (jetty-closer-11032-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@438bb23c: rootName = solr_43318, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@6c55c94a
   [junit4]   2> 4068683 INFO  (jetty-closer-11032-thread-1) [    ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:43318_solr
   [junit4]   2> 4068684 INFO  (jetty-closer-11032-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 4068684 INFO  (jetty-closer-11032-thread-1) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 4068684 INFO  (jetty-closer-11032-thread-1) [    ] o.a.s.c.Overseer Overseer (id=72068813957758979-127.0.0.1:43318_solr-n_0000000000) closing
   [junit4]   2> 4068685 INFO  (OverseerStateUpdate-72068813957758979-127.0.0.1:43318_solr-n_0000000000) [n:127.0.0.1:43318_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:43318_solr
   [junit4]   2> 4068689 INFO  (zkCallback-11041-thread-1-processing-n:127.0.0.1:43318_solr) [n:127.0.0.1:43318_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
   [junit4]   2> 4068687 WARN  (OverseerAutoScalingTriggerThread-72068813957758979-127.0.0.1:43318_solr-n_0000000000) [n:127.0.0.1:43318_solr    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 4068689 INFO  (zkCallback-11046-thread-1-processing-n:127.0.0.1:43318_solr) [n:127.0.0.1:43318_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
   [junit4]   2> 4068689 DEBUG (jetty-closer-11032-thread-1) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 4068696 INFO  (jetty-closer-11032-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@39bb82cd{/solr,null,UNAVAILABLE}
   [junit4]   2> 4068701 INFO  (jetty-closer-11032-thread-1) [    ] o.e.j.s.session Stopped scavenging
   [junit4]   2> 4068703 ERROR (SUITE-ScheduledTriggerTest-seed#[16B355D9DBFA6A65]-worker) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 4068704 INFO  (SUITE-ScheduledTriggerTest-seed#[16B355D9DBFA6A65]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:50783 50783
   [junit4]   2> 4069607 INFO  (Thread-10832) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:50783 50783
   [junit4]   2> 4069620 WARN  (Thread-10832) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	3	/solr/aliases.json
   [junit4]   2> 	3	/solr/clusterprops.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	3	/solr/clusterstate.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	3	/solr/live_nodes
   [junit4]   2> 	3	/solr/collections
   [junit4]   2> 
   [junit4]   2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.ScheduledTriggerTest_16B355D9DBFA6A65-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=1341, maxMBSortInHeap=5.385787755099965, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@118fbc79), locale=id-ID, timezone=America/Lima
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_162 (64-bit)/cpus=3,threads=1,free=134922552,total=536870912
   [junit4]   2> NOTE: All tests run in this JVM: [TestStressCloudBlindAtomicUpdates, TestRawResponseWriter, ZkControllerTest, SyncSliceTest, LoggingHandlerTest, TestPolicyCloud, SmileWriterTest, TestPayloadCheckQParserPlugin, AddBlockUpdateTest, DistributedFacetPivotSmallTest, ImplicitSnitchTest, TestCollectionAPI, TestLegacyNumericRangeQueryBuilder, TestAddFieldRealTimeGet, TestUniqueKeyFieldResource, SearchHandlerTest, UpdateRequestProcessorFactoryTest, CSVRequestHandlerTest, TestExtendedDismaxParser, ConnectionManagerTest, DistributedTermsComponentTest, TestMissingGroups, TestSolrCoreParser, ConcurrentCreateRoutedAliasTest, TestDistributedMissingSort, DistanceUnitsTest, TestTolerantUpdateProcessorCloud, TestFieldCache, NodeAddedTriggerTest, TestStressLucene, MinimalSchemaTest, TestCustomStream, AssignTest, TestFieldCacheVsDocValues, OverseerTest, TestStressReorder, TestEmbeddedSolrServerAdminHandler, HighlighterMaxOffsetTest, CdcrReplicationDistributedZkTest, DistributedDebugComponentTest, TestReplicaProperties, HdfsRecoverLeaseTest, URLClassifyProcessorTest, TestNumericTerms64, ClusterStateUpdateTest, TestManagedSchemaAPI, TestEmbeddedSolrServerSchemaAPI, TestFilteredDocIdSet, UpdateParamsTest, TestQuerySenderListener, TestRandomDVFaceting, AutoScalingHandlerTest, TriLevelCompositeIdRoutingTest, TestHashQParserPlugin, RankQueryTest, DirectoryFactoryTest, ParsingFieldUpdateProcessorsTest, TestSort, TestLRUCache, TestPHPSerializedResponseWriter, AtomicUpdateProcessorFactoryTest, TermVectorComponentTest, TestCollectionAPIs, TestFieldCacheSort, TestLegacyNumericUtils, TestValueSourceCache, ChaosMonkeyNothingIsSafeTest, DistribDocExpirationUpdateProcessorTest, TestDistribStateManager, TestFastLRUCache, AddReplicaTest, DistributedQueueTest, MetricsHandlerTest, TestSchemaResource, ChangedSchemaMergeTest, CollectionStateFormat2Test, ConnectionReuseTest, TestCloudPseudoReturnFields, DeleteLastCustomShardedReplicaTest, TestMultiValuedNumericRangeQuery, SoftAutoCommitTest, TestXmlQParserPlugin, TestJsonFacetsWithNestedObjects, TestNoOpRegenerator, SpellCheckCollatorWithCollapseTest, TestComplexPhraseLeadingWildcard, TokenizerChainTest, SuggestComponentContextFilterQueryTest, TestCSVLoader, CoreAdminHandlerTest, CdcrRequestHandlerTest, LeaderVoteWaitTimeoutTest, TestRestManager, BlockJoinFacetRandomTest, TestInfoStreamLogging, ClassificationUpdateProcessorTest, OverseerModifyCollectionTest, AnalyticsMergeStrategyTest, RestartWhileUpdatingTest, TestTestInjection, ScriptEngineTest, DistributedSpellCheckComponentTest, FieldAnalysisRequestHandlerTest, TestGenericDistributedQueue, ZkSolrClientTest, CdcrUpdateLogTest, TestSearcherReuse, RemoteQueryErrorTest, TestIndexingPerformance, DistributedSuggestComponentTest, TestHdfsCloudBackupRestore, TestConfigReload, TimeZoneUtilsTest, HdfsLockFactoryTest, SpellCheckCollatorTest, TestSchemalessBufferedUpdates, TestRandomFlRTGCloud, EchoParamsTest, DistributedQueryComponentCustomSortTest, AssignBackwardCompatibilityTest, CdcrBidirectionalTest, HdfsUnloadDistributedZkTest, TestAtomicUpdateErrorCases, DefaultValueUpdateProcessorTest, ClusterStateTest, TestSolrQueryParser, StressHdfsTest, ExternalFileFieldSortTest, TestNRTOpen, TestCoreDiscovery, DeleteReplicaTest, TestConfigSetsAPIZkFailure, TestSchemaVersionResource, BlockCacheTest, ExecutePlanActionTest, TestCloudSchemaless, ShardRoutingTest, TestRTGBase, TestQueryWrapperFilter, TestMaxTokenLenTokenizer, DOMUtilTest, InfixSuggestersTest, PeerSyncReplicationTest, TestLeaderElectionZkExpiry, TestConfigSets, HDFSCollectionsAPITest, SystemInfoHandlerTest, DisMaxRequestHandlerTest, TestDFRSimilarityFactory, TestRetrieveFieldsOptimizer, TestConfigSetImmutable, SolrCmdDistributorTest, TestLegacyField, EnumFieldTest, TestStressRecovery, TestNonDefinedSimilarityFactory, DistributedIntervalFacetingTest, SliceStateTest, TestStressLiveNodes, UUIDFieldTest, TestBulkSchemaConcurrent, BasicFunctionalityTest, TestRandomFaceting, TestWordDelimiterFilterFactory, ScheduledMaintenanceTriggerTest, MoveReplicaTest, TestDocSet, BadCopyFieldTest, HdfsThreadLeakTest, SolrCoreCheckLockOnStartupTest, NoCacheHeaderTest, LeaderFailoverAfterPartitionTest, SharedFSAutoReplicaFailoverTest, VersionInfoTest, V2StandaloneTest, LIRRollingUpdatesTest, TestInPlaceUpdatesStandalone, TestNumericRangeQuery64, TestCodecSupport, ExplicitHLLTest, TestLegacyFieldCache, TestStressVersions, TestCloudDeleteByQuery, HdfsDirectoryFactoryTest, TestStandardQParsers, TestSmileRequest, TestExclusionRuleCollectionAccess, PKIAuthenticationIntegrationTest, TestReqParamsAPI, ResponseLogComponentTest, BasicDistributedZkTest, ChaosMonkeySafeLeaderTest, BasicDistributedZk2Test, UnloadDistributedZkTest, OpenCloseCoreStressTest, LeaderElectionIntegrationTest, BasicZkTest, RecoveryZkTest, FullSolrCloudDistribCmdsTest, LeaderElectionTest, TestZkChroot, ZkCLITest, TestDistributedSearch, TestDistributedGrouping, TestFaceting, TestRecovery, TestHashPartitioner, TestRangeQuery, TestCoreContainer, SimpleFacetsTest, TestSolr4Spatial, StatsComponentTest, SpellCheckComponentTest, TestGroupingSearch, QueryElevationComponentTest, PeerSyncTest, BadIndexSchemaTest, TestFiltering, TestFunctionQuery, DirectUpdateHandlerTest, TestBadConfig, HighlighterTest, ShowFileRequestHandlerTest, SolrIndexSplitterTest, SimplePostToolTest, SignatureUpdateProcessorFactoryTest, SuggesterFSTTest, SolrRequestParserTest, TestFoldingMultitermQuery, DocValuesTest, SuggesterTSTTest, SuggesterTest, SpatialFilterTest, SuggesterWFSTTest, PolyFieldTest, WordBreakSolrSpellCheckerTest, SchemaVersionSpecificBehaviorTest, TestPseudoReturnFields, TestUpdate, FieldMutatingUpdateProcessorTest, QueryEqualityTest, StatelessScriptUpdateProcessorFactoryTest, SortByFunctionTest, TestRemoteStreaming, DocValuesMultiTest, DistanceFunctionTest, TestSolrDeletionPolicy1, SolrInfoBeanTest, XsltUpdateRequestHandlerTest, DebugComponentTest, IndexBasedSpellCheckerTest, TestSurroundQueryParser, TestQueryUtils, TestWriterPerf, DirectSolrSpellCheckerTest, TestReversedWildcardFilterFactory, DocumentAnalysisRequestHandlerTest, TestQueryTypes, PrimitiveFieldTypeTest, FileBasedSpellCheckerTest, RequestHandlersTest, DocumentBuilderTest, PathHierarchyTokenizerFactoryTest, RequiredFieldsTest, FastVectorHighlighterTest, RegexBoostProcessorTest, SolrPluginUtilsTest, IndexSchemaTest, TestCollationField, ReturnFieldsTest, TestCSVResponseWriter, MBeansHandlerTest, QueryParsingTest, TestAnalyzedSuggestions, JsonLoaderTest, TestPartialUpdateDeduplication, PingRequestHandlerTest, TestComponentsName, SearchHandlerTest, TestLFUCache, HighlighterConfigTest, TestSolrIndexConfig, BadComponentTest, TestSolrDeletionPolicy2, SampleTest, TestSearchPerf, NumericFieldsTest, TestConfig, TestFuzzyAnalyzedSuggestions, TestSolrCoreProperties, DirectSolrConnectionTest, NotRequiredUniqueKeyTest, TestLuceneMatchVersion, SpellPossibilityIteratorTest, TestCharFilters, TestXIncludeConfig, TestSweetSpotSimilarityFactory, TestLMDirichletSimilarityFactory, TestLMJelinekMercerSimilarityFactory, TestIBSimilarityFactory, TestBM25SimilarityFactory, TestFastWriter, ResourceLoaderTest, OpenExchangeRatesOrgProviderTest, PluginInfoTest, DateMathParserTest, PreAnalyzedFieldTest, PrimUtilsTest, TestSuggestSpellingConverter, DateFieldTest, TestDocumentBuilder, CircularListTest, CursorPagingTest, TestCrossCoreJoin, TestSimpleTrackingShardHandler, ActionThrottleTest, AliasIntegrationTest, ChaosMonkeyNothingIsSafeWithPullReplicasTest, ChaosMonkeySafeLeaderWithPullReplicasTest, CloudExitableDirectoryReaderTest, CollectionPropsTest, CollectionsAPISolrJTest, CreateRoutedAliasTest, DeleteInactiveReplicaTest, DeleteShardTest, DeleteStatusTest, DistributedVersionInfoTest, DocValuesNotIndexedTest, HealthCheckHandlerTest, HttpPartitionTest, LIROnShardRestartTest, LeaderFailureAfterFreshStartTest, LegacyCloudClusterPropTest, MigrateRouteKeyTest, MissingSegmentRecoveryTest, MoveReplicaHDFSFailoverTest, MoveReplicaHDFSTest, NodeMutatorTest, OutOfBoxZkACLAndCredentialsProvidersTest, OverriddenZkACLAndCredentialsProvidersTest, OverseerStatusTest, OverseerTaskQueueTest, ReplaceNodeNoTargetTest, ReplaceNodeTest, SSLMigrationTest, SaslZkACLProviderTest, SolrCLIZkUtilsTest, SolrCloudExampleTest, SolrXmlInZkTest, TestAuthenticationFramework, TestCloudConsistency, TestCloudJSONFacetJoinDomain, TestCloudPivotFacet, TestCloudRecovery, TestCloudSearcherWarming, TestClusterProperties, TestConfigSetsAPI, TestDeleteCollectionOnDownNodes, TestDistribDocBasedVersion, TestDistributedMap, TestLeaderInitiatedRecoveryThread, TestLockTree, TestOnReconnectListenerSupport, TestPullReplica, TestRebalanceLeaders, TestRequestForwarding, TestSSLRandomization, TestShortCircuitedRequests, TestSkipOverseerOperations, TestSolrCloudWithKerberosAlt, TestUtilizeNode, VMParamsZkACLAndCredentialsProvidersTest, ZkFailoverTest, ZkShardTermsTest, CollectionTooManyReplicasTest, CollectionsAPIAsyncDistributedZkTest, CollectionsAPIDistributedZkTest, CustomCollectionTest, HdfsCollectionsAPIDistributedZkTest, ShardSplitTest, TestCollectionsAPIViaSolrCloudCluster, AutoAddReplicasIntegrationTest, AutoAddReplicasPlanActionTest, ComputePlanActionTest, HdfsAutoAddReplicasIntegrationTest, HttpTriggerListenerTest, MetricTriggerTest, NodeLostTriggerTest, ScheduledTriggerTest]
   [junit4] Completed [398/785 (1!)] on J1 in 10.79s, 2 tests, 1 failure <<< FAILURES!

[...truncated 692 lines...]
   [junit4] Suite: org.apache.solr.core.TestLazyCores
   [junit4]   2> 11257 INFO  (SUITE-TestLazyCores-seed#[16B355D9DBFA6A65]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/init-core-data-001
   [junit4]   2> 11259 WARN  (SUITE-TestLazyCores-seed#[16B355D9DBFA6A65]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=10 numCloses=10
   [junit4]   2> 11260 INFO  (SUITE-TestLazyCores-seed#[16B355D9DBFA6A65]-worker) [    ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 11262 INFO  (SUITE-TestLazyCores-seed#[16B355D9DBFA6A65]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 11264 INFO  (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testCreateSame
   [junit4]   2> 11286 INFO  (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-001/solr.xml
   [junit4]   2> 11291 INFO  (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 11292 INFO  (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 11374 INFO  (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 11458 INFO  (coreLoadExecutor-72-thread-3) [    x:collection5] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 11473 INFO  (coreLoadExecutor-72-thread-2) [    x:collection2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 11478 INFO  (coreLoadExecutor-72-thread-3) [    x:collection5] o.a.s.s.IndexSchema [collection5] Schema name=tiny
   [junit4]   2> 11485 INFO  (coreLoadExecutor-72-thread-3) [    x:collection5] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 11485 INFO  (coreLoadExecutor-72-thread-3) [    x:collection5] o.a.s.c.CoreContainer Creating SolrCore 'collection5' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-001/./collection5, trusted=true
   [junit4]   2> 11494 INFO  (coreLoadExecutor-72-thread-3) [    x:collection5] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 11494 INFO  (coreLoadExecutor-72-thread-3) [    x:collection5] o.a.s.c.SolrCore [[collection5] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-001/collection5], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-001/./collection5/data/]
   [junit4]   2> 11497 INFO  (coreLoadExecutor-72-thread-3) [    x:collection5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=24, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 11498 INFO  (coreLoadExecutor-72-thread-2) [    x:collection2] o.a.s.s.IndexSchema [collection2] Schema name=tiny
   [junit4]   2> 11504 INFO  (coreLoadExecutor-72-thread-1) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 11513 INFO  (coreLoadExecutor-72-thread-2) [    x:collection2] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 11515 INFO  (coreLoadExecutor-72-thread-2) [    x:collection2] o.a.s.c.CoreContainer Creating SolrCore 'collection2' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-001/./collection2, trusted=true
   [junit4]   2> 11515 INFO  (coreLoadExecutor-72-thread-2) [    x:collection2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 11515 INFO  (coreLoadExecutor-72-thread-2) [    x:collection2] o.a.s.c.SolrCore [[collection2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-001/collection2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-001/./collection2/data/]
   [junit4]   2> 11520 INFO  (coreLoadExecutor-72-thread-2) [    x:collection2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=24, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 11526 INFO  (coreLoadExecutor-72-thread-1) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=tiny
   [junit4]   2> 11529 INFO  (coreLoadExecutor-72-thread-1) [    x:collection1] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 11529 INFO  (coreLoadExecutor-72-thread-1) [    x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-001/./collection1, trusted=true
   [junit4]   2> 11530 INFO  (coreLoadExecutor-72-thread-1) [    x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 11530 INFO  (coreLoadExecutor-72-thread-1) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-001/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-001/./collection1/data/]
   [junit4]   2> 11532 INFO  (coreLoadExecutor-72-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=24, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 11722 INFO  (coreLoadExecutor-72-thread-2) [    x:collection2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 11722 INFO  (coreLoadExecutor-72-thread-2) [    x:collection2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 11723 INFO  (coreLoadExecutor-72-thread-2) [    x:collection2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=16, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 11751 INFO  (coreLoadExecutor-72-thread-2) [    x:collection2] o.a.s.s.SolrIndexSearcher Opening [Searcher@4b4aef1b[collection2] main]
   [junit4]   2> 11752 INFO  (coreLoadExecutor-72-thread-2) [    x:collection2] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-001/collection2/conf
   [junit4]   2> 11753 INFO  (coreLoadExecutor-72-thread-2) [    x:collection2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 11759 INFO  (searcherExecutor-74-thread-1-processing-x:collection2) [    x:collection2] o.a.s.c.SolrCore [collection2] Registered new searcher Searcher@4b4aef1b[collection2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 11775 INFO  (coreLoadExecutor-72-thread-1) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 11775 INFO  (coreLoadExecutor-72-thread-1) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 11776 INFO  (coreLoadExecutor-72-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=16, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 11778 INFO  (coreLoadExecutor-72-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@415e3684[collection1] main]
   [junit4]   2> 11778 INFO  (coreLoadExecutor-72-thread-1) [    x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-001/collection1/conf
   [junit4]   2> 11779 INFO  (coreLoadExecutor-72-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 11782 INFO  (searcherExecutor-75-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@415e3684[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 11793 INFO  (coreLoadExecutor-72-thread-3) [    x:collection5] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 11793 INFO  (coreLoadExecutor-72-thread-3) [    x:collection5] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 11794 INFO  (coreLoadExecutor-72-thread-3) [    x:collection5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=16, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 11796 INFO  (coreLoadExecutor-72-thread-3) [    x:collection5] o.a.s.s.SolrIndexSearcher Opening [Searcher@57d7f404[collection5] main]
   [junit4]   2> 11799 INFO  (coreLoadExecutor-72-thread-3) [    x:collection5] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-001/collection5/conf
   [junit4]   2> 11800 INFO  (coreLoadExecutor-72-thread-3) [    x:collection5] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 11802 INFO  (searcherExecutor-73-thread-1-processing-x:collection5) [    x:collection5] o.a.s.c.SolrCore [collection5] Registered new searcher Searcher@57d7f404[collection5] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 11820 INFO  (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 11837 INFO  (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.s.IndexSchema [collection4] Schema name=tiny
   [junit4]   2> 11841 INFO  (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 11841 INFO  (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.c.CoreContainer Creating SolrCore 'collection4' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-001/./collection4, trusted=true
   [junit4]   2> 11841 INFO  (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 11841 INFO  (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.c.SolrCore [[collection4] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-001/collection4], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-001/./collection4/data/]
   [junit4]   2> 11846 INFO  (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=45, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 11945 INFO  (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 11945 INFO  (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 11946 INFO  (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=47, maxMergeAtOnceExplicit=47, maxMergedSegmentMB=45.869140625, floorSegmentMB=1.318359375, forceMergeDeletesPctAllowed=18.995916436538693, segmentsPerTier=15.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 11947 INFO  (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.s.SolrIndexSearcher Opening [Searcher@3604b76b[collection4] main]
   [junit4]   2> 11948 INFO  (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-001/collection4/conf
   [junit4]   2> 11948 INFO  (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 11950 INFO  (searcherExecutor-86-thread-1-processing-x:collection4) [    x:collection4] o.a.s.c.SolrCore [collection4] Registered new searcher Searcher@3604b76b[collection4] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 11968 INFO  (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 11981 INFO  (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.s.IndexSchema [collection6] Schema name=tiny
   [junit4]   2> 11986 INFO  (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 11986 INFO  (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.c.CoreContainer Creating SolrCore 'collection6' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-001/./collection6, trusted=true
   [junit4]   2> 11986 INFO  (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 11986 INFO  (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.c.SolrCore [[collection6] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-001/collection6], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-001/./collection6/data/]
   [junit4]   2> 11989 INFO  (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=24, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 12133 INFO  (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 12133 INFO  (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 12135 INFO  (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1014728150, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 12140 INFO  (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.s.SolrIndexSearcher Opening [Searcher@19cfe4c8[collection6] main]
   [junit4]   2> 12141 INFO  (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-001/collection6/conf
   [junit4]   2> 12141 INFO  (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 12159 INFO  (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.h.a.CoreAdminOperation core create command schema=schema.xml&dataDir=t12&name=collection2&action=CREATE&config=solrconfig.xml&wt=xml
   [junit4]   2> 12159 WARN  (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.c.CoreContainer Creating a core with existing name is not allowed
   [junit4]   2> 12160 INFO  (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.h.a.CoreAdminOperation core create command schema=schema.xml&dataDir=t14&name=collection4&action=CREATE&config=solrconfig.xml&wt=xml
   [junit4]   2> 12160 WARN  (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.c.CoreContainer Creating a core with existing name is not allowed
   [junit4]   2> 12160 INFO  (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.h.a.CoreAdminOperation core create command schema=schema.xml&dataDir=t15&name=collection5&action=CREATE&config=solrconfig.xml&wt=xml
   [junit4]   2> 12160 WARN  (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.c.CoreContainer Creating a core with existing name is not allowed
   [junit4]   2> 12160 INFO  (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.h.a.CoreAdminOperation core create command schema=schema.xml&dataDir=t16&name=collection6&action=CREATE&config=solrconfig.xml&wt=xml
   [junit4]   2> 12188 WARN  (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.c.CoreContainer Creating a core with existing name is not allowed
   [junit4]   2> 12188 INFO  (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.c.CoreContainer Shutting down CoreContainer instance=511387418
   [junit4]   2> 12188 INFO  (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 12188 INFO  (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 12189 INFO  (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 12189 INFO  (searcherExecutor-90-thread-1-processing-x:collection4) [    x:collection4] o.a.s.c.SolrCore [collection6] Registered new searcher Searcher@19cfe4c8[collection6] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 12191 INFO  (coreCloseExecutor-94-thread-1-processing-x:collection4) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@fa0a3d8
   [junit4]   2> 12191 INFO  (coreCloseExecutor-94-thread-1-processing-x:collection4) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=262185944
   [junit4]   2> 12209 INFO  (coreCloseExecutor-94-thread-4-processing-x:collection4) [    x:collection2] o.a.s.c.SolrCore [collection2]  CLOSING SolrCore org.apache.solr.core.SolrCore@2ff4053c
   [junit4]   2> 12209 INFO  (coreCloseExecutor-94-thread-4-processing-x:collection4) [    x:collection2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection2, tag=804521276
   [junit4]   2> 12210 INFO  (coreCloseExecutor-94-thread-2-processing-x:collection4) [    x:collection5] o.a.s.c.SolrCore [collection5]  CLOSING SolrCore org.apache.solr.core.SolrCore@113d3f5c
   [junit4]   2> 12210 INFO  (coreCloseExecutor-94-thread-2-processing-x:collection4) [    x:collection5] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection5, tag=289226588
   [junit4]   2> 12212 INFO  (coreCloseExecutor-94-thread-3-processing-x:collection4) [    x:collection4] o.a.s.c.SolrCore [collection4]  CLOSING SolrCore org.apache.solr.core.SolrCore@69aa006b
   [junit4]   2> 12212 INFO  (coreCloseExecutor-94-thread-3-processing-x:collection4) [    x:collection4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection4, tag=1772748907
   [junit4]   2> 12212 INFO  (coreCloseExecutor-94-thread-5-processing-x:collection4) [    x:collection6] o.a.s.c.SolrCore [collection6]  CLOSING SolrCore org.apache.solr.core.SolrCore@794bf524
   [junit4]   2> 12212 INFO  (coreCloseExecutor-94-thread-5-processing-x:collection4) [    x:collection6] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection6, tag=2035021092
   [junit4]   2> 12226 INFO  (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.SolrTestCaseJ4 ###Ending testCreateSame
   [junit4]   2> 12229 INFO  (TEST-TestLazyCores.testMidUseUnload-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.SolrTestCaseJ4 ###Starting testMidUseUnload
   [junit4]   2> 12253 INFO  (TEST-TestLazyCores.testMidUseUnload-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.c.SolrXmlConfig Loading container configuration from /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-002/solr.xml
   [junit4]   2> 12258 INFO  (TEST-TestLazyCores.testMidUseUnload-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 12258 INFO  (TEST-TestLazyCores.testMidUseUnload-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 12301 INFO  (TEST-TestLazyCores.testMidUseUnload-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 12333 INFO  (coreLoadExecutor-101-thread-2-processing-x:collection4) [    x:collection2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 12393 INFO  (coreLoadExecutor-101-thread-1-processing-x:collection4) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 12402 INFO  (coreLoadExecutor-101-thread-1-processing-x:collection4) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=tiny
   [junit4]   2> 12414 INFO  (coreLoadExecutor-101-thread-2-processing-x:collection4) [    x:collection2] o.a.s.s.IndexSchema [collection2] Schema name=tiny
   [junit4]   2> 12417 INFO  (coreLoadExecutor-101-thread-2-processing-x:collection4) [    x:collection2] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 12417 INFO  (coreLoadExecutor-101-thread-2-processing-x:collection4) [    x:collection2] o.a.s.c.CoreContainer Creating SolrCore 'collection2' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-002/./collection2, trusted=true
   [junit4]   2> 12417 INFO  (coreLoadExecutor-101-thread-2-processing-x:collection4) [    x:collection2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 12417 INFO  (coreLoadExecutor-101-thread-2-processing-x:collection4) [    x:collection2] o.a.s.c.SolrCore [[collection2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-002/collection2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-002/./collection2/data/]
   [junit4]   2> 12420 INFO  (coreLoadExecutor-101-thread-2-processing-x:collection4) [    x:collection2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=24, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 12422 INFO  (coreLoadExecutor-101-thread-1-processing-x:collection4) [    x:collection1] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 12422 INFO  (coreLoadExecutor-101-thread-1-processing-x:collection4) [    x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-002/./collection1, trusted=true
   [junit4]   2> 12422 INFO  (coreLoadExecutor-101-thread-1-processing-x:collection4) [    x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 12422 INFO  (coreLoadExecutor-101-thread-1-processing-x:collection4) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-002/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-002/./collection1/data/]
   [junit4]   2> 12425 INFO  (coreLoadExecutor-101-thread-1-processing-x:collection4) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=24, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 12432 INFO  (coreLoadExecutor-101-thread-3-processing-x:collection4) [    x:collection5] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 12490 INFO  (coreLoadExecutor-101-thread-2-processing-x:collection4) [    x:collection2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 12490 INFO  (coreLoadExecutor-101-thread-2-processing-x:collection4) [    x:collection2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 12491 INFO  (coreLoadExecutor-101-thread-2-processing-x:collection4) [    x:collection2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=16, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 12536 INFO  (coreLoadExecutor-101-thread-1-processing-x:collection4) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 12536 INFO  (coreLoadExecutor-101-thread-1-processing-x:collection4) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 12537 INFO  (coreLoadExecutor-101-thread-1-processing-x:collection4) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=16, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 12546 INFO  (coreLoadExecutor-101-thread-3-processing-x:collection4) [    x:collection5] o.a.s.s.IndexSchema [collection5] Schema name=tiny
   [junit4]   2> 12550 INFO  (coreLoadExecutor-101-thread-3-processing-x:collection4) [    x:collection5] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 12550 INFO  (coreLoadExecutor-101-thread-3-processing-x:collection4) [    x:collection5] o.a.s.c.CoreContainer Creating SolrCore 'collection5' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-002/./collection5, trusted=true
   [junit4]   2> 12550 INFO  (coreLoadExecutor-101-thread-3-processing-x:collection4) [    x:collection5] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 12550 INFO  (coreLoadExecutor-101-thread-3-processing-x:collection4) [    x:collection5] o.a.s.c.SolrCore [[collection5] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-002/collection5], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-002/./collection5/data/]
   [junit4]   2> 12552 INFO  (coreLoadExecutor-101-thread-3-processing-x:collection4) [    x:collection5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=24, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 12554 INFO  (coreLoadExecutor-101-thread-1-processing-x:collection4) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@85e1127[collection1] main]
   [junit4]   2> 12554 INFO  (coreLoadExecutor-101-thread-1-processing-x:collection4) [    x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-002/collection1/conf
   [junit4]   2> 12555 INFO  (coreLoadExecutor-101-thread-1-processing-x:collection4) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 12560 INFO  (searcherExecutor-103-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@85e1127[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 12567 INFO  (coreLoadExecutor-101-thread-2-processing-x:collection4) [    x:collection2] o.a.s.s.SolrIndexSearcher Opening [Searcher@70be7fd3[collection2] main]
   [junit4]   2> 12568 INFO  (coreLoadExecutor-101-thread-2-processing-x:collection4) [    x:collection2] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-002/collection2/conf
   [junit4]   2> 12569 INFO  (coreLoadExecutor-101-thread-2-processing-x:collection4) [    x:collection2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 12570 INFO  (searcherExecutor-102-thread-1-processing-x:collection2) [    x:collection2] o.a.s.c.SolrCore [collection2] Registered new searcher Searcher@70be7fd3[collection2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 12629 INFO  (coreLoadExecutor-101-thread-3-processing-x:collection4) [    x:collection5] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 12629 INFO  (coreLoadExecutor-101-thread-3-processing-x:collection4) [    x:collection5] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 12630 INFO  (coreLoadExecutor-101-thread-3-processing-x:collection4) [    x:collection5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=16, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 12631 INFO  (coreLoadExecutor-101-thread-3-processing-x:collection4) [    x:collection5] o.a.s.s.SolrIndexSearcher Opening [Searcher@5fdf2bcf[collection5] main]
   [junit4]   2> 12631 INFO  (coreLoadExecutor-101-thread-3-processing-x:collection4) [    x:collection5] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-002/collection5/conf
   [junit4]   2> 12632 INFO  (coreLoadExecutor-101-thread-3-processing-x:collection4) [    x:collection5] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 12633 INFO  (searcherExecutor-110-thread-1-processing-x:collection5) [    x:collection5] o.a.s.c.SolrCore [collection5] Registered new searcher Searcher@5fdf2bcf[collection5] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 12638 INFO  (TEST-TestLazyCores.testMidUseUnload-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=null
   [junit4]   2> 12639 INFO  (TEST-TestLazyCores.testMidUseUnload-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.c.SolrCore Core collection1 is not yet closed, waiting 100 ms before checking again.
   [junit4]   2> 12739 INFO  (TEST-TestLazyCores.testMidUseUnload-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.c.SolrCore Core collection1 is not yet closed, waiting 100 ms before checking again.
   [junit4]   2> 12839 INFO  (TEST-TestLazyCores.testMidUseUnload-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.c.SolrCore Core collection1 is not yet closed, waiting 100 ms before checking again.
   [junit4]   2> 12942 INFO  (TEST-TestLazyCores.testMidUseUnload-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.c.SolrCore Core collection1 is not yet closed, waiting 100 ms before checking again.
   [junit4]   2> 13043 INFO  (TEST-TestLazyCores.testMidUseUnload-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.c.SolrCore Core collection1 is not yet closed, waiting 100 ms before checking again.
   [junit4]   2> 13147 INFO  (TEST-TestLazyCores.testMidUseUnload-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.c.SolrCore Core collection1 is not yet closed, waiting 100 ms before checking again.
   [junit4]   2> 13248 INFO  (TEST-TestLazyCores.testMidUseUnload-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.c.SolrCore Core collection1 is not yet closed, waiting 100 ms before checking again.
   [junit4]   2> 13348 INFO  (TEST-TestLazyCores.testMidUseUnload-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.c.SolrCore Core collection1 is not yet closed, waiting 100 ms before checking again.
   [junit4]   2> 13438 INFO  (Thread-58) [    x:collection4] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@40d0716f
   [junit4]   2> 13438 INFO  (Thread-58) [    x:collection4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=1087402351
   [junit4]   2> 13448 INFO  (TEST-TestLazyCores.testMidUseUnload-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1613191220
   [junit4]   2> 13448 INFO  (TEST-TestLazyCores.testMidUseUnload-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 13448 INFO  (TEST-TestLazyCores.testMidUseUnload-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 13448 INFO  (TEST-TestLazyCores.testMidUseUnload-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 13460 INFO  (coreCloseExecutor-115-thread-1-processing-x:collection4) [    x:collection5] o.a.s.c.SolrCore [collection5]  CLOSING SolrCore org.apache.solr.core.SolrCore@265e60f0
   [junit4]   2> 13460 INFO  (coreCloseExecutor-115-thread-1-processing-x:collection4) [    x:collection5] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection5, tag=643719408
   [junit4]   2> 13460 INFO  (coreCloseExecutor-115-thread-2-processing-x:collection4) [    x:collection2] o.a.s.c.SolrCore [collection2]  CLOSING SolrCore org.apache.solr.core.SolrCore@6d5ce0b8
   [junit4]   2> 13460 INFO  (coreCloseExecutor-115-thread-2-processing-x:collection4) [    x:collection2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection2, tag=1834803384
   [junit4]   2> 13466 INFO  (TEST-TestLazyCores.testMidUseUnload-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.SolrTestCaseJ4 ###Ending testMidUseUnload
   [junit4]   2> 13489 INFO  (TEST-TestLazyCores.testRace-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.SolrTestCaseJ4 ###Starting testRace
   [junit4]   2> 13507 INFO  (TEST-TestLazyCores.testRace-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.c.SolrXmlConfig Loading container configuration from /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-003/solr.xml
   [junit4]   2> 13512 INFO  (TEST-TestLazyCores.testRace-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 13512 INFO  (TEST-TestLazyCores.testRace-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 13577 INFO  (TEST-TestLazyCores.testRace-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 13610 INFO  (coreLoadExecutor-122-thread-2-processing-x:collection4) [    x:collection2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 13622 INFO  (coreLoadExecutor-122-thread-2-processing-x:collection4) [    x:collection2] o.a.s.s.IndexSchema [collection2] Schema name=tiny
   [junit4]   2> 13625 INFO  (coreLoadExecutor-122-thread-2-processing-x:collection4) [    x:collection2] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 13625 INFO  (coreLoadExecutor-122-thread-2-processing-x:collection4) [    x:collection2] o.a.s.c.CoreContainer Creating SolrCore 'collection2' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-003/./collection2, trusted=true
   [junit4]   2> 13626 INFO  (coreLoadExecutor-122-thread-2-processing-x:collection4) [    x:collection2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 13626 INFO  (coreLoadExecutor-122-thread-2-processing-x:collection4) [    x:collection2] o.a.s.c.SolrCore [[collection2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-003/collection2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-003/./collection2/data/]
   [junit4]   2> 13614 INFO  (coreLoadExecutor-122-thread-1-processing-x:collection4) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 13639 INFO  (coreLoadExecutor-122-thread-1-processing-x:collection4) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=tiny
   [junit4]   2> 13641 INFO  (coreLoadExecutor-122-thread-1-processing-x:collection4) [    x:collection1] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 13641 INFO  (coreLoadExecutor-122-thread-1-processing-x:collection4) [    x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-003/./collection1, trusted=true
   [junit4]   2> 13642 INFO  (coreLoadExecutor-122-thread-1-processing-x:collection4) [    x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 13642 INFO  (coreLoadExecutor-122-thread-1-processing-x:collection4) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-003/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-003/./collection1/data/]
   [junit4]   2> 13645 INFO  (coreLoadExecutor-122-thread-2-processing-x:collection4) [    x:collection2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=24, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 13646 INFO  (coreLoadExecutor-122-thread-1-processing-x:collection4) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=24, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 13651 INFO  (coreLoadExecutor-122-thread-3-processing-x:collection4) [    x:collection5] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 13668 INFO  (coreLoadExecutor-122-thread-3-processing-x:collection4) [    x:collection5] o.a.s.s.IndexSchema [collection5] Schema name=tiny
   [junit4]   2> 13670 INFO  (coreLoadExecutor-122-thread-3-processing-x:collection4) [    x:collection5] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 13670 INFO  (coreLoadExecutor-122-thread-3-processing-x:collection4) [    x:collection5] o.a.s.c.CoreContainer Creating SolrCore 'collection5' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-003/./collection5, trusted=true
   [junit4]   2> 13672 INFO  (coreLoadExecutor-122-thread-3-processing-x:collection4) [    x:collection5] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 13672 INFO  (coreLoadExecutor-122-thread-3-processing-x:collection4) [    x:collection5] o.a.s.c.SolrCore [[collection5] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-003/collection5], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-003/./collection5/data/]
   [junit4]   2> 13677 INFO  (coreLoadExecutor-122-thread-3-processing-x:collection4) [    x:collection5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=24, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 14038 INFO  (coreLoadExecutor-122-thread-1-processing-x:collection4) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 14038 INFO  (coreLoadExecutor-122-thread-1-processing-x:collection4) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 14038 INFO  (coreLoadExecutor-122-thread-1-processing-x:collection4) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=16, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 14053 INFO  (coreLoadExecutor-122-thread-1-processing-x:collection4) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@56e5c311[collection1] main]
   [junit4]   2> 14129 INFO  (coreLoadExecutor-122-thread-2-processing-x:collection4) [    x:collection2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 14129 INFO  (coreLoadExecutor-122-thread-2-processing-x:collection4) [    x:collection2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 14130 INFO  (coreLoadExecutor-122-thread-2-processing-x:collection4) [    x:collection2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=16, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 14166 INFO  (coreLoadExecutor-122-thread-3-processing-x:collection4) [    x:collection5] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 14166 INFO  (coreLoadExecutor-122-thread-3-processing-x:collection4) [    x:collection5] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 14166 INFO  (coreLoadExecutor-122-thread-3-processing-x:collection4) [    x:collection5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=16, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 14167 INFO  (coreLoadExecutor-122-thread-1-processing-x:collection4) [    x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-003/collection1/conf
   [junit4]   2> 14167 INFO  (coreLoadExecutor-122-thread-1-processing-x:collection4) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 14168 INFO  (coreLoadExecutor-122-thread-2-processing-x:collection4) [    x:collection2] o.a.s.s.SolrIndexSearcher Opening [Searcher@11807a5b[collection2] main]
   [junit4]   2> 14169 INFO  (coreLoadExecutor-122-thread-2-processing-x:collection4) [    x:collection2] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-003/collection2/conf
   [junit4]   2> 14169 INFO  (coreLoadExecutor-122-thread-2-processing-x:collection4) [    x:collection2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 14170 INFO  (coreLoadExecutor-122-thread-3-processing-x:collection4) [    x:collection5] o.a.s.s.SolrIndexSearcher Opening [Searcher@5739a46e[collection5] main]
   [junit4]   2> 14171 INFO  (coreLoadExecutor-122-thread-3-processing-x:collection4) [    x:collection5] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-003/collection5/conf
   [junit4]   2> 14171 INFO  (coreLoadExecutor-122-thread-3-processing-x:collection4) [    x:collection5] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 14194 INFO  (searcherExecutor-124-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@56e5c311[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 14195 INFO  (searcherExecutor-123-thread-1-processing-x:collection2) [    x:collection2] o.a.s.c.SolrCore [collection2] Registered new searcher Searcher@11807a5b[collection2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 14216 INFO  (Thread-74) [    x:collection3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 14251 INFO  (searcherExecutor-125-thread-1-processing-x:collection5) [    x:collection5] o.a.s.c.SolrCore [collection5] Registered new searcher Searcher@5739a46e[collection5] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 14256 INFO  (Thread-74) [    x:collection3] o.a.s.s.IndexSchema [collection3] Schema name=tiny
   [junit4]   2> 14286 INFO  (Thread-74) [    x:collection3] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 14286 INFO  (Thread-74) [    x:collection3] o.a.s.c.CoreContainer Creating SolrCore 'collection3' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-003/./collection3, trusted=true
   [junit4]   2> 14286 INFO  (Thread-74) [    x:collection3] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 14286 INFO  (Thread-74) [    x:collection3] o.a.s.c.SolrCore [[collection3] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-003/collection3], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-003/./collection3/data/]
   [junit4]   2> 14299 INFO  (Thread-74) [    x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=24, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 14408 INFO  (Thread-74) [    x:collection3] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 14408 INFO  (Thread-74) [    x:collection3] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 14408 INFO  (Thread-74) [    x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=16, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 14410 INFO  (Thread-74) [  

[...truncated too long message...]

o.a.s.c.SolrCore [collection2] Registered new searcher Searcher@59ef38aa[collection2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 7304286 INFO  (TEST-TestLazyCores.testLazySearch-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1177801611
   [junit4]   2> 7304286 WARN  (TEST-TestLazyCores.testLazySearch-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.m.SolrMetricManager Interrupted while trying to obtain lock to modify reporters registry: solr.node
   [junit4]   2> 7304286 INFO  (TEST-TestLazyCores.testLazySearch-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 7304286 INFO  (TEST-TestLazyCores.testLazySearch-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@748ccd53: rootName = null, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@670997af
   [junit4]   2> 7304288 INFO  (TEST-TestLazyCores.testLazySearch-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 7304288 INFO  (TEST-TestLazyCores.testLazySearch-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@525466f: rootName = null, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@69f1cfd3
   [junit4]   2> 7304289 INFO  (coreCloseExecutor-192-thread-1-processing-x:collection4) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@7d090299
   [junit4]   2> 7304289 INFO  (coreCloseExecutor-192-thread-1-processing-x:collection4) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=2097742489
   [junit4]   2> 7304289 INFO  (coreCloseExecutor-192-thread-1-processing-x:collection4) [    x:collection1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@56a1385e: rootName = null, domain = solr.core.collection1, service url = null, agent id = null] for registry solr.core.collection1 / com.codahale.metrics.MetricRegistry@71a9294e
   [junit4]   2> 7304289 INFO  (coreCloseExecutor-192-thread-2-processing-x:collection4) [    x:collection5] o.a.s.c.SolrCore [collection5]  CLOSING SolrCore org.apache.solr.core.SolrCore@fbbed29
   [junit4]   2> 7304289 INFO  (coreCloseExecutor-192-thread-3-processing-x:collection4) [    x:collection2] o.a.s.c.SolrCore [collection2]  CLOSING SolrCore org.apache.solr.core.SolrCore@776e593b
   [junit4]   2> 7304294 INFO  (coreCloseExecutor-192-thread-2-processing-x:collection4) [    x:collection5] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection5, tag=263974185
   [junit4]   2> 7304294 INFO  (coreCloseExecutor-192-thread-2-processing-x:collection4) [    x:collection5] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@17fe3705: rootName = null, domain = solr.core.collection5, service url = null, agent id = null] for registry solr.core.collection5 / com.codahale.metrics.MetricRegistry@61470c75
   [junit4]   2> 7304302 INFO  (coreCloseExecutor-192-thread-3-processing-x:collection4) [    x:collection2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection2, tag=2003720507
   [junit4]   2> 7304302 INFO  (coreCloseExecutor-192-thread-3-processing-x:collection4) [    x:collection2] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@69ef3bd4: rootName = null, domain = solr.core.collection2, service url = null, agent id = null] for registry solr.core.collection2 / com.codahale.metrics.MetricRegistry@7fb25968
   [junit4]   2> 7304311 INFO  (TEST-TestLazyCores.testLazySearch-seed#[16B355D9DBFA6A65]) [    x:collection4] o.a.s.SolrTestCaseJ4 ###Ending testLazySearch
   [junit4]   2> 7304312 INFO  (SUITE-TestLazyCores-seed#[16B355D9DBFA6A65]-worker) [    x:collection4] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> 7304312 INFO  (SUITE-TestLazyCores-seed#[16B355D9DBFA6A65]-worker) [    x:collection4] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1177801611
   [junit4]   2> 7304312 INFO  (SUITE-TestLazyCores-seed#[16B355D9DBFA6A65]-worker) [    x:collection4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 7304312 INFO  (SUITE-TestLazyCores-seed#[16B355D9DBFA6A65]-worker) [    x:collection4] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@5247cc41: rootName = null, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@6b537520
   [junit4]   2> 7304315 INFO  (SUITE-TestLazyCores-seed#[16B355D9DBFA6A65]-worker) [    x:collection4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 7304315 INFO  (SUITE-TestLazyCores-seed#[16B355D9DBFA6A65]-worker) [    x:collection4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001
   [junit4]   2> Mar 16, 2018 1:13:11 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> SEVERE: There are still zombie threads that couldn't be terminated:
   [junit4]   2>    1) Thread[id=191, name=searcherExecutor-136-thread-1, state=WAITING, group=TGRP-TestLazyCores]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]   2>         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {v_t=BlockTreeOrds(blocksize=128), id=BlockTreeOrds(blocksize=128), v_s=Lucene50(blocksize=128), v_f=PostingsFormat(name=LuceneVarGapFixedInterval)}, docValues:{}, maxPointsInLeafNode=179, maxMBSortInHeap=6.340553083811743, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@6a1f220c), locale=id-ID, timezone=America/Lima
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_162 (64-bit)/cpus=3,threads=2,free=174467576,total=359661568
   [junit4]   2> NOTE: All tests run in this JVM: [ScheduledTriggerTest, TestLazyCores]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLazyCores -Dtests.seed=16B355D9DBFA6A65 -Dtests.slow=true -Dtests.badapples=true -Dtests.locale=id-ID -Dtests.timezone=America/Lima -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J1 | TestLazyCores (suite) <<<
   [junit4]    > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([16B355D9DBFA6A65]:0)
   [junit4] Completed [8/10 (1!)] on J1 in 7283.30s, 6 tests, 2 errors <<< FAILURES!

[...truncated 17 lines...]
BUILD FAILED
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/lucene/common-build.xml:1561: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/lucene/common-build.xml:1085: There were test failures: 10 suites, 52 tests, 1 suite-level error, 1 error, 4 ignored [seed: 16B355D9DBFA6A65]

Total time: 121 minutes 54 seconds

[repro] Setting last failure code to 256

[repro] Failures:
[repro]   0/5 failed: org.apache.solr.cloud.autoscaling.ScheduledTriggerTest
[repro]   1/5 failed: org.apache.solr.core.TestLazyCores
[repro] Exiting with code 256
+ mv lucene/build lucene/build.repro
+ mv solr/build solr/build.repro
+ mv lucene/build.orig lucene/build
+ mv solr/build.orig solr/build
Archiving artifacts
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
[WARNINGS] Parsing warnings in console log with parser Java Compiler (javac)
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
<Git Blamer> Using GitBlamer to create author and commit information for all warnings.
<Git Blamer> GIT_COMMIT=2ca741d36a3078e7d7b03cb73176a1e99377eefc, workspace=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris
[WARNINGS] Computing warning deltas based on reference build #1735
Recording test results
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2

[JENKINS] Lucene-Solr-master-Solaris (64bit/jdk1.8.0) - Build # 1735 - Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1735/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseParallelGC

3 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.DeleteLastCustomShardedReplicaTest

Error Message:
java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:41409 within 30000 ms

Stack Trace:
org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:41409 within 30000 ms
	at __randomizedtesting.SeedInfo.seed([DF44788DBE20EF01]:0)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:183)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:120)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:115)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:102)
	at org.apache.solr.cloud.MiniSolrCloudCluster.waitForAllNodes(MiniSolrCloudCluster.java:268)
	at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:262)
	at org.apache.solr.cloud.SolrCloudTestCase$Builder.configure(SolrCloudTestCase.java:190)
	at org.apache.solr.cloud.DeleteLastCustomShardedReplicaTest.setupCluster(DeleteLastCustomShardedReplicaTest.java:31)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:874)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:41409 within 30000 ms
	at org.apache.solr.common.cloud.ConnectionManager.waitForConnected(ConnectionManager.java:232)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:175)
	... 31 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.DeleteLastCustomShardedReplicaTest

Error Message:
28 threads leaked from SUITE scope at org.apache.solr.cloud.DeleteLastCustomShardedReplicaTest:     1) Thread[id=7335, name=qtp1329013312-7335-acceptor-0@ac94e40-ServerConnector@247acfac{HTTP/1.1,[http/1.1]}{127.0.0.1:58786}, state=RUNNABLE, group=TGRP-DeleteLastCustomShardedReplicaTest]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:379)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:638)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=7363, name=qtp1661555970-7363, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=7327, name=ProcessThread(sid:0 cport:41409):, state=WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:123)    4) Thread[id=7347, name=qtp1661555970-7347, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=7325, name=SessionTracker, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]         at java.lang.Object.wait(Native Method)         at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:147)    6) Thread[id=7355, name=qtp1329013312-7355, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)         at java.lang.Thread.run(Thread.java:748)    7) Thread[id=7324, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-DeleteLastCustomShardedReplicaTest]         at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)         at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)         at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:196)         at java.lang.Thread.run(Thread.java:748)    8) Thread[id=7348, name=qtp1661555970-7348, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163)         at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308)         at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)    9) Thread[id=7334, name=qtp1329013312-7334, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)         at java.lang.Thread.run(Thread.java:748)   10) Thread[id=7360, name=qtp1661555970-7360, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)         at java.lang.Thread.run(Thread.java:748)   11) Thread[id=7364, name=qtp1661555970-7364, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)         at java.lang.Thread.run(Thread.java:748)   12) Thread[id=7350, name=qtp1661555970-7350-acceptor-0@1bfd09c6-ServerConnector@3c4c0ab3{HTTP/1.1,[http/1.1]}{127.0.0.1:41409}, state=RUNNABLE, group=TGRP-DeleteLastCustomShardedReplicaTest]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:379)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:638)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   13) Thread[id=7361, name=qtp1661555970-7361, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)         at java.lang.Thread.run(Thread.java:748)   14) Thread[id=7333, name=qtp1329013312-7333, state=RUNNABLE, group=TGRP-DeleteLastCustomShardedReplicaTest]         at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)         at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)         at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$28/664856532.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   15) Thread[id=7362, name=Scheduler-1842259731, state=WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   16) Thread[id=7323, name=Thread-2090, state=WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]         at java.lang.Object.wait(Native Method)         at java.lang.Thread.join(Thread.java:1252)         at java.lang.Thread.join(Thread.java:1326)         at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:313)         at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:310)         at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:491)   17) Thread[id=7344, name=qtp1661555970-7344, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)         at java.lang.Thread.run(Thread.java:748)   18) Thread[id=7336, name=qtp1329013312-7336, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)         at java.lang.Thread.run(Thread.java:748)   19) Thread[id=7340, name=qtp1329013312-7340, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)         at java.lang.Thread.run(Thread.java:748)   20) Thread[id=7357, name=qtp1329013312-7357, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)         at java.lang.Thread.run(Thread.java:748)   21) Thread[id=7358, name=qtp1661555970-7358, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)         at java.lang.Thread.run(Thread.java:748)   22) Thread[id=7349, name=qtp1661555970-7349, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)         at java.lang.Thread.run(Thread.java:748)   23) Thread[id=7345, name=qtp1661555970-7345, state=RUNNABLE, group=TGRP-DeleteLastCustomShardedReplicaTest]         at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)         at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)         at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)         at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   24) Thread[id=7346, name=qtp1661555970-7346, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)         at java.lang.Thread.run(Thread.java:748)   25) Thread[id=7352, name=Scheduler-2081218491, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   26) Thread[id=7338, name=qtp1329013312-7338, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)         at java.lang.Thread.run(Thread.java:748)   27) Thread[id=7341, name=Scheduler-959118455, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   28) Thread[id=7356, name=qtp1329013312-7356, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 28 threads leaked from SUITE scope at org.apache.solr.cloud.DeleteLastCustomShardedReplicaTest: 
   1) Thread[id=7335, name=qtp1329013312-7335-acceptor-0@ac94e40-ServerConnector@247acfac{HTTP/1.1,[http/1.1]}{127.0.0.1:58786}, state=RUNNABLE, group=TGRP-DeleteLastCustomShardedReplicaTest]
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
        at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:379)
        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:638)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=7363, name=qtp1661555970-7363, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=7327, name=ProcessThread(sid:0 cport:41409):, state=WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:123)
   4) Thread[id=7347, name=qtp1661555970-7347, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=7325, name=SessionTracker, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
        at java.lang.Object.wait(Native Method)
        at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:147)
   6) Thread[id=7355, name=qtp1329013312-7355, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
        at java.lang.Thread.run(Thread.java:748)
   7) Thread[id=7324, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-DeleteLastCustomShardedReplicaTest]
        at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
        at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
        at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:196)
        at java.lang.Thread.run(Thread.java:748)
   8) Thread[id=7348, name=qtp1661555970-7348, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163)
        at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308)
        at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
   9) Thread[id=7334, name=qtp1329013312-7334, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
        at java.lang.Thread.run(Thread.java:748)
  10) Thread[id=7360, name=qtp1661555970-7360, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
        at java.lang.Thread.run(Thread.java:748)
  11) Thread[id=7364, name=qtp1661555970-7364, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
        at java.lang.Thread.run(Thread.java:748)
  12) Thread[id=7350, name=qtp1661555970-7350-acceptor-0@1bfd09c6-ServerConnector@3c4c0ab3{HTTP/1.1,[http/1.1]}{127.0.0.1:41409}, state=RUNNABLE, group=TGRP-DeleteLastCustomShardedReplicaTest]
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
        at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:379)
        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:638)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  13) Thread[id=7361, name=qtp1661555970-7361, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
        at java.lang.Thread.run(Thread.java:748)
  14) Thread[id=7333, name=qtp1329013312-7333, state=RUNNABLE, group=TGRP-DeleteLastCustomShardedReplicaTest]
        at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
        at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
        at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$28/664856532.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  15) Thread[id=7362, name=Scheduler-1842259731, state=WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  16) Thread[id=7323, name=Thread-2090, state=WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Thread.join(Thread.java:1252)
        at java.lang.Thread.join(Thread.java:1326)
        at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:313)
        at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:310)
        at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:491)
  17) Thread[id=7344, name=qtp1661555970-7344, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
        at java.lang.Thread.run(Thread.java:748)
  18) Thread[id=7336, name=qtp1329013312-7336, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
        at java.lang.Thread.run(Thread.java:748)
  19) Thread[id=7340, name=qtp1329013312-7340, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
        at java.lang.Thread.run(Thread.java:748)
  20) Thread[id=7357, name=qtp1329013312-7357, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
        at java.lang.Thread.run(Thread.java:748)
  21) Thread[id=7358, name=qtp1661555970-7358, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
        at java.lang.Thread.run(Thread.java:748)
  22) Thread[id=7349, name=qtp1661555970-7349, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
        at java.lang.Thread.run(Thread.java:748)
  23) Thread[id=7345, name=qtp1661555970-7345, state=RUNNABLE, group=TGRP-DeleteLastCustomShardedReplicaTest]
        at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
        at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
        at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)
        at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  24) Thread[id=7346, name=qtp1661555970-7346, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
        at java.lang.Thread.run(Thread.java:748)
  25) Thread[id=7352, name=Scheduler-2081218491, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  26) Thread[id=7338, name=qtp1329013312-7338, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
        at java.lang.Thread.run(Thread.java:748)
  27) Thread[id=7341, name=Scheduler-959118455, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  28) Thread[id=7356, name=qtp1329013312-7356, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([DF44788DBE20EF01]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.DeleteLastCustomShardedReplicaTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=7333, name=qtp1329013312-7333, state=RUNNABLE, group=TGRP-DeleteLastCustomShardedReplicaTest]         at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)         at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)         at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$28/664856532.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=7362, name=Scheduler-1842259731, state=WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=7345, name=qtp1661555970-7345, state=RUNNABLE, group=TGRP-DeleteLastCustomShardedReplicaTest]         at sun.nio.ch.IOUtil.drain(Native Method)         at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:109)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)         at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=7324, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-DeleteLastCustomShardedReplicaTest]         at sun.nio.ch.DevPollArrayWrapper.interrupt(Native Method)         at sun.nio.ch.DevPollArrayWrapper.interrupt(DevPollArrayWrapper.java:300)         at sun.nio.ch.DevPollSelectorImpl.wakeup(DevPollSelectorImpl.java:207)         at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)         at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)         at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:196)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=7352, name=Scheduler-2081218491, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=7348, name=qtp1661555970-7348, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163)         at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308)         at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)    7) Thread[id=7341, name=Scheduler-959118455, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=7333, name=qtp1329013312-7333, state=RUNNABLE, group=TGRP-DeleteLastCustomShardedReplicaTest]
        at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
        at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
        at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$28/664856532.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=7362, name=Scheduler-1842259731, state=WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=7345, name=qtp1661555970-7345, state=RUNNABLE, group=TGRP-DeleteLastCustomShardedReplicaTest]
        at sun.nio.ch.IOUtil.drain(Native Method)
        at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:109)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)
        at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=7324, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-DeleteLastCustomShardedReplicaTest]
        at sun.nio.ch.DevPollArrayWrapper.interrupt(Native Method)
        at sun.nio.ch.DevPollArrayWrapper.interrupt(DevPollArrayWrapper.java:300)
        at sun.nio.ch.DevPollSelectorImpl.wakeup(DevPollSelectorImpl.java:207)
        at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
        at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
        at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:196)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=7352, name=Scheduler-2081218491, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=7348, name=qtp1661555970-7348, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163)
        at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308)
        at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
   7) Thread[id=7341, name=Scheduler-959118455, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([DF44788DBE20EF01]:0)




Build Log:
[...truncated 1837 lines...]
   [junit4] JVM J1: stdout was not empty, see: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/lucene/build/core/test/temp/junit4-J1-20180315_231150_6945423026239327529097.sysout
   [junit4] >>> JVM J1 emitted unexpected output (verbatim) ----
   [junit4] codec: CheapBastard, pf: LuceneVarGapFixedInterval, dvf: Memory
   [junit4] <<< JVM J1: EOF ----

[...truncated 10810 lines...]
   [junit4] Suite: org.apache.solr.cloud.DeleteLastCustomShardedReplicaTest
   [junit4]   2> 605087 INFO  (SUITE-DeleteLastCustomShardedReplicaTest-seed#[DF44788DBE20EF01]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.DeleteLastCustomShardedReplicaTest_DF44788DBE20EF01-001/init-core-data-001
   [junit4]   2> 605087 WARN  (SUITE-DeleteLastCustomShardedReplicaTest-seed#[DF44788DBE20EF01]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=2 numCloses=2
   [junit4]   2> 605087 INFO  (SUITE-DeleteLastCustomShardedReplicaTest-seed#[DF44788DBE20EF01]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 605089 INFO  (SUITE-DeleteLastCustomShardedReplicaTest-seed#[DF44788DBE20EF01]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 605090 INFO  (SUITE-DeleteLastCustomShardedReplicaTest-seed#[DF44788DBE20EF01]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.DeleteLastCustomShardedReplicaTest_DF44788DBE20EF01-001/tempDir-001
   [junit4]   2> 605090 INFO  (SUITE-DeleteLastCustomShardedReplicaTest-seed#[DF44788DBE20EF01]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 605090 INFO  (Thread-2090) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 605090 INFO  (Thread-2090) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 605091 ERROR (Thread-2090) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 605190 INFO  (SUITE-DeleteLastCustomShardedReplicaTest-seed#[DF44788DBE20EF01]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:41409
   [junit4]   2> 605193 INFO  (zkConnectionManagerCallback-1666-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 605199 INFO  (jetty-launcher-1663-thread-1) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T15:27:37-06:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 605200 INFO  (jetty-launcher-1663-thread-1) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 605200 INFO  (jetty-launcher-1663-thread-1) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 605200 INFO  (jetty-launcher-1663-thread-1) [    ] o.e.j.s.session Scavenging every 600000ms
   [junit4]   2> 605201 INFO  (jetty-launcher-1663-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@18062087{/solr,null,AVAILABLE}
   [junit4]   2> 605202 INFO  (jetty-launcher-1663-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@247acfac{HTTP/1.1,[http/1.1]}{127.0.0.1:58786}
   [junit4]   2> 605202 INFO  (jetty-launcher-1663-thread-1) [    ] o.e.j.s.Server Started @607305ms
   [junit4]   2> 605202 INFO  (jetty-launcher-1663-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=58786}
   [junit4]   2> 605203 ERROR (jetty-launcher-1663-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 605203 INFO  (jetty-launcher-1663-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 605203 INFO  (jetty-launcher-1663-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 605203 INFO  (jetty-launcher-1663-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 605203 INFO  (jetty-launcher-1663-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-03-15T23:37:05.398Z
   [junit4]   2> 605205 INFO  (jetty-launcher-1663-thread-2) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T15:27:37-06:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 605207 INFO  (jetty-launcher-1663-thread-2) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 605207 INFO  (jetty-launcher-1663-thread-2) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 605207 INFO  (jetty-launcher-1663-thread-2) [    ] o.e.j.s.session Scavenging every 660000ms
   [junit4]   2> 605207 INFO  (jetty-launcher-1663-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@32173193{/solr,null,AVAILABLE}
   [junit4]   2> 605208 INFO  (jetty-launcher-1663-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@3c4c0ab3{HTTP/1.1,[http/1.1]}{127.0.0.1:41409}
   [junit4]   2> 605208 INFO  (jetty-launcher-1663-thread-2) [    ] o.e.j.s.Server Started @607311ms
   [junit4]   2> 605208 INFO  (jetty-launcher-1663-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=41409}
   [junit4]   2> 605208 ERROR (jetty-launcher-1663-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 605208 INFO  (jetty-launcher-1663-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 605208 INFO  (jetty-launcher-1663-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 605208 INFO  (jetty-launcher-1663-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 605208 INFO  (jetty-launcher-1663-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-03-15T23:37:05.403Z
   [junit4]   2> 605214 WARN  (qtp1661555970-7347) [    ] o.e.j.h.HttpParser Illegal character 0x0 in state=START for buffer HeapByteBuffer@f923d2f[p=1,l=49,c=8192,r=48]={\x00<<<\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00>>>\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00}
   [junit4]   2> 605216 WARN  (jetty-launcher-1663-thread-2-SendThread(127.0.0.1:41409)) [    ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:41409, unexpected error, closing socket connection and attempting reconnect
   [junit4]   2> java.io.IOException: Packet len1213486160 is out of range!
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:113)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1144)
   [junit4]   2> 605224 WARN  (qtp1661555970-7345) [    ] o.e.j.h.HttpParser Illegal character 0x0 in state=START for buffer HeapByteBuffer@f923d2f[p=1,l=49,c=8192,r=48]={\x00<<<\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00>>>\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00}
   [junit4]   2> 605225 WARN  (jetty-launcher-1663-thread-1-SendThread(127.0.0.1:41409)) [    ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:41409, unexpected error, closing socket connection and attempting reconnect
   [junit4]   2> java.io.IOException: Packet len1213486160 is out of range!
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:113)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1144)
   [junit4]   2> 606349 WARN  (qtp1661555970-7344) [    ] o.e.j.h.HttpParser Illegal character 0x0 in state=START for buffer HeapByteBuffer@f923d2f[p=1,l=49,c=8192,r=48]={\x00<<<\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00>>>\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00}
   [junit4]   2> 606349 WARN  (jetty-launcher-1663-thread-1-SendThread(127.0.0.1:41409)) [    ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:41409, unexpected error, closing socket connection and attempting reconnect
   [junit4]   2> java.io.IOException: Packet len1213486160 is out of range!
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:113)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1144)
   [junit4]   2> 607246 WARN  (qtp1661555970-7345) [    ] o.e.j.h.HttpParser Illegal character 0x0 in state=START for buffer HeapByteBuffer@f923d2f[p=1,l=49,c=8192,r=48]={\x00<<<\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00>>>\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00}
   [junit4]   2> 607247 WARN  (jetty-launcher-1663-thread-2-SendThread(127.0.0.1:41409)) [    ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:41409, unexpected error, closing socket connection and attempting reconnect
   [junit4]   2> java.io.IOException: Packet len1213486160 is out of range!
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:113)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1144)
   [junit4]   2> 607903 WARN  (qtp1661555970-7344) [    ] o.e.j.h.HttpParser Illegal character 0x0 in state=START for buffer HeapByteBuffer@f923d2f[p=1,l=49,c=8192,r=48]={\x00<<<\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00>>>\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00}
   [junit4]   2> 607904 WARN  (jetty-launcher-1663-thread-1-SendThread(127.0.0.1:41409)) [    ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:41409, unexpected error, closing socket connection and attempting reconnect
   [junit4]   2> java.io.IOException: Packet len1213486160 is out of range!
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:113)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1144)
   [junit4]   2> 609260 WARN  (qtp1661555970-7345) [    ] o.e.j.h.HttpParser Illegal character 0x0 in state=START for buffer HeapByteBuffer@f923d2f[p=1,l=49,c=8192,r=48]={\x00<<<\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00>>>\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00}
   [junit4]   2> 609260 WARN  (jetty-launcher-1663-thread-2-SendThread(127.0.0.1:41409)) [    ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:41409, unexpected error, closing socket connection and attempting reconnect
   [junit4]   2> java.io.IOException: Packet len1213486160 is out of range!
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:113)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1144)
   [junit4]   2> 609773 WARN  (qtp1661555970-7344) [    ] o.e.j.h.HttpParser Illegal character 0x0 in state=START for buffer HeapByteBuffer@f923d2f[p=1,l=49,c=8192,r=48]={\x00<<<\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00>>>\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00}
   [junit4]   2> 609774 WARN  (jetty-launcher-1663-thread-1-SendThread(127.0.0.1:41409)) [    ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:41409, unexpected error, closing socket connection and attempting reconnect
   [junit4]   2> java.io.IOException: Packet len1213486160 is out of range!
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:113)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1144)
   [junit4]   2> 610925 WARN  (qtp1661555970-7345) [    ] o.e.j.h.HttpParser Illegal character 0x0 in state=START for buffer HeapByteBuffer@f923d2f[p=1,l=49,c=8192,r=48]={\x00<<<\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00>>>\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00}
   [junit4]   2> 610926 WARN  (jetty-launcher-1663-thread-2-SendThread(127.0.0.1:41409)) [    ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:41409, unexpected error, closing socket connection and attempting reconnect
   [junit4]   2> java.io.IOException: Packet len1213486160 is out of range!
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:113)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1144)
   [junit4]   2> 611678 WARN  (qtp1661555970-7344) [    ] o.e.j.h.HttpParser Illegal character 0x0 in state=START for buffer HeapByteBuffer@f923d2f[p=1,l=49,c=8192,r=48]={\x00<<<\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00>>>\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00}
   [junit4]   2> 611678 WARN  (jetty-launcher-1663-thread-1-SendThread(127.0.0.1:41409)) [    ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:41409, unexpected error, closing socket connection and attempting reconnect
   [junit4]   2> java.io.IOException: Packet len1213486160 is out of range!
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:113)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1144)
   [junit4]   2> 612239 WARN  (qtp1661555970-7345) [    ] o.e.j.h.HttpParser Illegal character 0x0 in state=START for buffer HeapByteBuffer@f923d2f[p=1,l=49,c=8192,r=48]={\x00<<<\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00>>>\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00}
   [junit4]   2> 612239 WARN  (jetty-launcher-1663-thread-2-SendThread(127.0.0.1:41409)) [    ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:41409, unexpected error, closing socket connection and attempting reconnect
   [junit4]   2> java.io.IOException: Packet len1213486160 is out of range!
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:113)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1144)
   [junit4]   2> 613081 WARN  (qtp1661555970-7344) [    ] o.e.j.h.HttpParser Illegal character 0x0 in state=START for buffer HeapByteBuffer@f923d2f[p=1,l=49,c=8192,r=48]={\x00<<<\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00>>>\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00}
   [junit4]   2> 613082 WARN  (jetty-launcher-1663-thread-1-SendThread(127.0.0.1:41409)) [    ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:41409, unexpected error, closing socket connection and attempting reconnect
   [junit4]   2> java.io.IOException: Packet len1213486160 is out of range!
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:113)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1144)
   [junit4]   2> 614327 WARN  (qtp1661555970-7345) [    ] o.e.j.h.HttpParser Illegal character 0x0 in state=START for buffer HeapByteBuffer@f923d2f[p=1,l=49,c=8192,r=48]={\x00<<<\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00>>>\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00}
   [junit4]   2> 614327 WARN  (jetty-launcher-1663-thread-1-SendThread(127.0.0.1:41409)) [    ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:41409, unexpected error, closing socket connection and attempting reconnect
   [junit4]   2> java.io.IOException: Packet len1213486160 is out of range!
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:113)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1144)
   [junit4]   2> 614335 WARN  (qtp1661555970-7344) [    ] o.e.j.h.HttpParser Illegal character 0x0 in state=START for buffer HeapByteBuffer@f923d2f[p=1,l=49,c=8192,r=48]={\x00<<<\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00>>>\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00}
   [junit4]   2> 614335 WARN  (jetty-launcher-1663-thread-2-SendThread(127.0.0.1:41409)) [    ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:41409, unexpected error, closing socket connection and attempting reconnect
   [junit4]   2> java.io.IOException: Packet len1213486160 is out of range!
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:113)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1144)
   [junit4]   2> 615902 WARN  (qtp1661555970-7345) [    ] o.e.j.h.HttpParser Illegal character 0x0 in state=START for buffer HeapByteBuffer@f923d2f[p=1,l=49,c=8192,r=48]={\x00<<<\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00>>>\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00}
   [junit4]   2> 615902 WARN  (jetty-launcher-1663-thread-1-SendThread(127.0.0.1:41409)) [    ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:41409, unexpected error, closing socket connection and attempting reconnect
   [junit4]   2> java.io.IOException: Packet len1213486160 is out of range!
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:113)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1144)
   [junit4]   2> 616396 WARN  (qtp1661555970-7344) [    ] o.e.j.h.HttpParser Illegal character 0x0 in state=START for buffer HeapByteBuffer@f923d2f[p=1,l=49,c=8192,r=48]={\x00<<<\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00>>>\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00}
   [junit4]   2> 616397 WARN  (jetty-launcher-1663-thread-2-SendThread(127.0.0.1:41409)) [    ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:41409, unexpected error, closing socket connection and attempting reconnect
   [junit4]   2> java.io.IOException: Packet len1213486160 is out of range!
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:113)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1144)
   [junit4]   2> 617643 WARN  (qtp1661555970-7345) [    ] o.e.j.h.HttpParser Illegal character 0x0 in state=START for buffer HeapByteBuffer@f923d2f[p=1,l=49,c=8192,r=48]={\x00<<<\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00>>>\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00}
   [junit4]   2> 617643 WARN  (jetty-launcher-1663-thread-2-SendThread(127.0.0.1:41409)) [    ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:41409, unexpected error, closing socket connection and attempting reconnect
   [junit4]   2> java.io.IOException: Packet len1213486160 is out of range!
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:113)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1144)
   [junit4]   2> 617893 WARN  (qtp1661555970-7344) [    ] o.e.j.h.HttpParser Illegal character 0x0 in state=START for buffer HeapByteBuffer@f923d2f[p=1,l=49,c=8192,r=48]={\x00<<<\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00>>>\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00}
   [junit4]   2> 617894 WARN  (jetty-launcher-1663-thread-1-SendThread(127.0.0.1:41409)) [    ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:41409, unexpected error, closing socket connection and attempting reconnect
   [junit4]   2> java.io.IOException: Packet len1213486160 is out of range!
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:113)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1144)
   [junit4]   2> 619404 WARN  (qtp1661555970-7345) [    ] o.e.j.h.HttpParser Illegal character 0x0 in state=START for buffer HeapByteBuffer@f923d2f[p=1,l=49,c=8192,r=48]={\x00

[...truncated too long message...]

 [junit4]    > 	at __randomizedtesting.SeedInfo.seed([DF44788DBE20EF01]:0)Throwable #3: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   [junit4]    >    1) Thread[id=7333, name=qtp1329013312-7333, state=RUNNABLE, group=TGRP-DeleteLastCustomShardedReplicaTest]
   [junit4]    >         at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
   [junit4]    >         at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
   [junit4]    >         at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
   [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$$Lambda$28/664856532.run(Unknown Source)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    2) Thread[id=7362, name=Scheduler-1842259731, state=WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    3) Thread[id=7345, name=qtp1661555970-7345, state=RUNNABLE, group=TGRP-DeleteLastCustomShardedReplicaTest]
   [junit4]    >         at sun.nio.ch.IOUtil.drain(Native Method)
   [junit4]    >         at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:109)
   [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)
   [junit4]    >         at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    4) Thread[id=7324, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-DeleteLastCustomShardedReplicaTest]
   [junit4]    >         at sun.nio.ch.DevPollArrayWrapper.interrupt(Native Method)
   [junit4]    >         at sun.nio.ch.DevPollArrayWrapper.interrupt(DevPollArrayWrapper.java:300)
   [junit4]    >         at sun.nio.ch.DevPollSelectorImpl.wakeup(DevPollSelectorImpl.java:207)
   [junit4]    >         at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
   [junit4]    >         at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
   [junit4]    >         at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:97)
   [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]    >         at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:196)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    5) Thread[id=7352, name=Scheduler-2081218491, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    6) Thread[id=7348, name=qtp1661555970-7348, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163)
   [junit4]    >         at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308)
   [junit4]    >         at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    7) Thread[id=7341, name=Scheduler-959118455, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([DF44788DBE20EF01]:0)
   [junit4] Completed [149/785 (1!)] on J1 in 151.05s, 0 tests, 3 errors <<< FAILURES!

[...truncated 51531 lines...]
[repro] Jenkins log URL: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1735/consoleText

[repro] Revision: ac9559d70de5e2454e0515a08774890ad4012eef

[repro] Ant options: "-Dargs=-XX:-UseCompressedOops -XX:+UseParallelGC"
[repro] Repro line:  ant test  -Dtestcase=DeleteLastCustomShardedReplicaTest -Dtests.seed=DF44788DBE20EF01 -Dtests.slow=true -Dtests.locale=en-PH -Dtests.timezone=Europe/Malta -Dtests.asserts=true -Dtests.file.encoding=US-ASCII

[repro] ant clean

[...truncated 6 lines...]
[repro] Test suites by module:
[repro]    solr/core
[repro]       DeleteLastCustomShardedReplicaTest
[repro] ant compile-test

[...truncated 3313 lines...]
[repro] ant test-nocompile -Dtests.dups=5 -Dtests.maxfailures=5 -Dtests.class="*.DeleteLastCustomShardedReplicaTest" -Dtests.showOutput=onerror "-Dargs=-XX:-UseCompressedOops -XX:+UseParallelGC" -Dtests.seed=DF44788DBE20EF01 -Dtests.slow=true -Dtests.locale=en-PH -Dtests.timezone=Europe/Malta -Dtests.asserts=true -Dtests.file.encoding=US-ASCII

[...truncated 68 lines...]
[repro] Failures:
[repro]   0/5 failed: org.apache.solr.cloud.DeleteLastCustomShardedReplicaTest
[repro] Exiting with code 0

[...truncated 70 lines...]