You are viewing a plain text version of this content. The canonical link for it is here.
Posted to builds@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2020/01/18 09:17:26 UTC

[JENKINS] Lucene-Solr-NightlyTests-master - Build # 2077 - Unstable

Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/2077/

1 tests failed.
FAILED:  org.apache.solr.cloud.hdfs.HdfsSyncSliceTest.test

Error Message:
expected:<5> but was:<4>

Stack Trace:
java.lang.AssertionError: expected:<5> but was:<4>
	at __randomizedtesting.SeedInfo.seed([8F0796456E3CE4D5:753A99FC0C0892D]:0)
	at org.junit.Assert.fail(Assert.java:88)
	at org.junit.Assert.failNotEquals(Assert.java:834)
	at org.junit.Assert.assertEquals(Assert.java:645)
	at org.junit.Assert.assertEquals(Assert.java:631)
	at org.apache.solr.cloud.SyncSliceTest.test(SyncSliceTest.java:150)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1082)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1054)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:834)




Build Log:
[...truncated 15247 lines...]
   [junit4] Suite: org.apache.solr.cloud.hdfs.HdfsSyncSliceTest
   [junit4]   2> 4911649 INFO  (SUITE-HdfsSyncSliceTest-seed#[8F0796456E3CE4D5]-worker) [     ] o.a.s.SolrTestCaseJ4 Created dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsSyncSliceTest_8F0796456E3CE4D5-001/data-dir-280-001
   [junit4]   2> 4911649 WARN  (SUITE-HdfsSyncSliceTest-seed#[8F0796456E3CE4D5]-worker) [     ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=8 numCloses=8
   [junit4]   2> 4911649 INFO  (SUITE-HdfsSyncSliceTest-seed#[8F0796456E3CE4D5]-worker) [     ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 4911650 INFO  (SUITE-HdfsSyncSliceTest-seed#[8F0796456E3CE4D5]-worker) [     ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason="", ssl=0.0/0.0, value=0.0/0.0, clientAuth=0.0/0.0)
   [junit4]   2> 4911651 INFO  (SUITE-HdfsSyncSliceTest-seed#[8F0796456E3CE4D5]-worker) [     ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 4911651 INFO  (SUITE-HdfsSyncSliceTest-seed#[8F0796456E3CE4D5]-worker) [     ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /zw_/
   [junit4]   1> Formatting using clusterid: testClusterID
   [junit4]   2> 4911910 WARN  (SUITE-HdfsSyncSliceTest-seed#[8F0796456E3CE4D5]-worker) [     ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 4911914 INFO  (SUITE-HdfsSyncSliceTest-seed#[8F0796456E3CE4D5]-worker) [     ] o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z; git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 11.0.4+10-LTS
   [junit4]   2> 4911916 INFO  (SUITE-HdfsSyncSliceTest-seed#[8F0796456E3CE4D5]-worker) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 4911916 INFO  (SUITE-HdfsSyncSliceTest-seed#[8F0796456E3CE4D5]-worker) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 4911916 INFO  (SUITE-HdfsSyncSliceTest-seed#[8F0796456E3CE4D5]-worker) [     ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 4911916 INFO  (SUITE-HdfsSyncSliceTest-seed#[8F0796456E3CE4D5]-worker) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@532ff958{static,/static,jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/static,AVAILABLE}
   [junit4]   2> 4912038 INFO  (SUITE-HdfsSyncSliceTest-seed#[8F0796456E3CE4D5]-worker) [     ] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@73233ef6{hdfs,/,file:///home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/jetty-localhost_localdomain-37399-hadoop-hdfs-3_2_0-tests_jar-_-any-4927007323145481382.dir/webapp/,AVAILABLE}{jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/hdfs}
   [junit4]   2> 4912038 INFO  (SUITE-HdfsSyncSliceTest-seed#[8F0796456E3CE4D5]-worker) [     ] o.e.j.s.AbstractConnector Started ServerConnector@4e8cb4d{HTTP/1.1,[http/1.1]}{localhost.localdomain:37399}
   [junit4]   2> 4912038 INFO  (SUITE-HdfsSyncSliceTest-seed#[8F0796456E3CE4D5]-worker) [     ] o.e.j.s.Server Started @4912064ms
   [junit4]   2> 4912197 WARN  (SUITE-HdfsSyncSliceTest-seed#[8F0796456E3CE4D5]-worker) [     ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 4912198 INFO  (SUITE-HdfsSyncSliceTest-seed#[8F0796456E3CE4D5]-worker) [     ] o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z; git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 11.0.4+10-LTS
   [junit4]   2> 4912199 INFO  (SUITE-HdfsSyncSliceTest-seed#[8F0796456E3CE4D5]-worker) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 4912199 INFO  (SUITE-HdfsSyncSliceTest-seed#[8F0796456E3CE4D5]-worker) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 4912199 INFO  (SUITE-HdfsSyncSliceTest-seed#[8F0796456E3CE4D5]-worker) [     ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 4912200 INFO  (SUITE-HdfsSyncSliceTest-seed#[8F0796456E3CE4D5]-worker) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1ed8f9da{static,/static,jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/static,AVAILABLE}
   [junit4]   2> 4912513 INFO  (SUITE-HdfsSyncSliceTest-seed#[8F0796456E3CE4D5]-worker) [     ] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@7c059e6{datanode,/,file:///home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/jetty-localhost-39245-hadoop-hdfs-3_2_0-tests_jar-_-any-16492770382009200808.dir/webapp/,AVAILABLE}{jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/datanode}
   [junit4]   2> 4912514 INFO  (SUITE-HdfsSyncSliceTest-seed#[8F0796456E3CE4D5]-worker) [     ] o.e.j.s.AbstractConnector Started ServerConnector@6476d43b{HTTP/1.1,[http/1.1]}{localhost:39245}
   [junit4]   2> 4912514 INFO  (SUITE-HdfsSyncSliceTest-seed#[8F0796456E3CE4D5]-worker) [     ] o.e.j.s.Server Started @4912540ms
   [junit4]   2> 4912653 WARN  (SUITE-HdfsSyncSliceTest-seed#[8F0796456E3CE4D5]-worker) [     ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 4912654 INFO  (SUITE-HdfsSyncSliceTest-seed#[8F0796456E3CE4D5]-worker) [     ] o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z; git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 11.0.4+10-LTS
   [junit4]   2> 4912676 INFO  (SUITE-HdfsSyncSliceTest-seed#[8F0796456E3CE4D5]-worker) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 4912676 INFO  (SUITE-HdfsSyncSliceTest-seed#[8F0796456E3CE4D5]-worker) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 4912676 INFO  (SUITE-HdfsSyncSliceTest-seed#[8F0796456E3CE4D5]-worker) [     ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 4912680 INFO  (SUITE-HdfsSyncSliceTest-seed#[8F0796456E3CE4D5]-worker) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6d559d9{static,/static,jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/static,AVAILABLE}
   [junit4]   2> 4913103 INFO  (SUITE-HdfsSyncSliceTest-seed#[8F0796456E3CE4D5]-worker) [     ] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@721d398b{datanode,/,file:///home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/jetty-localhost-34479-hadoop-hdfs-3_2_0-tests_jar-_-any-3917704253615675338.dir/webapp/,AVAILABLE}{jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/datanode}
   [junit4]   2> 4913104 INFO  (SUITE-HdfsSyncSliceTest-seed#[8F0796456E3CE4D5]-worker) [     ] o.e.j.s.AbstractConnector Started ServerConnector@598bf907{HTTP/1.1,[http/1.1]}{localhost:34479}
   [junit4]   2> 4913104 INFO  (SUITE-HdfsSyncSliceTest-seed#[8F0796456E3CE4D5]-worker) [     ] o.e.j.s.Server Started @4913130ms
   [junit4]   2> 4913132 INFO  (Block report processor) [     ] BlockStateChange BLOCK* processReport 0xd29cdb8beb38bcc2: Processing first storage report for DS-22288e7c-a9dc-440d-bd7e-b9f16b698518 from datanode 33ac210d-2c91-4cc6-90cd-76290ee12992
   [junit4]   2> 4913132 INFO  (Block report processor) [     ] BlockStateChange BLOCK* processReport 0xd29cdb8beb38bcc2: from storage DS-22288e7c-a9dc-440d-bd7e-b9f16b698518 node DatanodeRegistration(127.0.0.1:44425, datanodeUuid=33ac210d-2c91-4cc6-90cd-76290ee12992, infoPort=35477, infoSecurePort=0, ipcPort=40045, storageInfo=lv=-57;cid=testClusterID;nsid=762485349;c=1579336770771), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
   [junit4]   2> 4913132 INFO  (Block report processor) [     ] BlockStateChange BLOCK* processReport 0xd29cdb8beb38bcc2: Processing first storage report for DS-bc6dcd47-ea29-42fd-acc1-a4572e12ab5e from datanode 33ac210d-2c91-4cc6-90cd-76290ee12992
   [junit4]   2> 4913132 INFO  (Block report processor) [     ] BlockStateChange BLOCK* processReport 0xd29cdb8beb38bcc2: from storage DS-bc6dcd47-ea29-42fd-acc1-a4572e12ab5e node DatanodeRegistration(127.0.0.1:44425, datanodeUuid=33ac210d-2c91-4cc6-90cd-76290ee12992, infoPort=35477, infoSecurePort=0, ipcPort=40045, storageInfo=lv=-57;cid=testClusterID;nsid=762485349;c=1579336770771), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
   [junit4]   2> 4913370 INFO  (Block report processor) [     ] BlockStateChange BLOCK* processReport 0x6c08230bf4cc374b: Processing first storage report for DS-551a5eec-16fd-4450-ae6d-6de821097e47 from datanode c09c0221-e59b-4315-bef9-16c8db44b778
   [junit4]   2> 4913370 INFO  (Block report processor) [     ] BlockStateChange BLOCK* processReport 0x6c08230bf4cc374b: from storage DS-551a5eec-16fd-4450-ae6d-6de821097e47 node DatanodeRegistration(127.0.0.1:36515, datanodeUuid=c09c0221-e59b-4315-bef9-16c8db44b778, infoPort=33689, infoSecurePort=0, ipcPort=37511, storageInfo=lv=-57;cid=testClusterID;nsid=762485349;c=1579336770771), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
   [junit4]   2> 4913370 INFO  (Block report processor) [     ] BlockStateChange BLOCK* processReport 0x6c08230bf4cc374b: Processing first storage report for DS-04656c19-b94d-46c5-bd19-e33dad2e2cd2 from datanode c09c0221-e59b-4315-bef9-16c8db44b778
   [junit4]   2> 4913370 INFO  (Block report processor) [     ] BlockStateChange BLOCK* processReport 0x6c08230bf4cc374b: from storage DS-04656c19-b94d-46c5-bd19-e33dad2e2cd2 node DatanodeRegistration(127.0.0.1:36515, datanodeUuid=c09c0221-e59b-4315-bef9-16c8db44b778, infoPort=33689, infoSecurePort=0, ipcPort=37511, storageInfo=lv=-57;cid=testClusterID;nsid=762485349;c=1579336770771), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
   [junit4]   2> 4913509 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 4913510 INFO  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 4913510 INFO  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 4913610 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.ZkTestServer start zk server on port:45607
   [junit4]   2> 4913610 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.ZkTestServer waitForServerUp: 127.0.0.1:45607
   [junit4]   2> 4913610 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:45607
   [junit4]   2> 4913610 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 45607
   [junit4]   2> 4913690 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 4913697 INFO  (zkConnectionManagerCallback-27303-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4913697 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 4913702 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 4913704 INFO  (zkConnectionManagerCallback-27305-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4913704 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 4913705 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 4913707 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 4913710 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 4913712 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 4913714 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 4913715 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 4913716 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 4913717 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 4913719 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 4913721 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 4913722 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 4913726 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 4913727 INFO  (zkConnectionManagerCallback-27309-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4913727 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 4913831 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
   [junit4]   2> 4914025 WARN  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
   [junit4]   2> 4914025 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 4914025 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 4914025 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z; git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 11.0.4+10-LTS
   [junit4]   2> 4914171 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 4914171 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 4914171 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 4914174 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@75600db8{/zw_,null,AVAILABLE}
   [junit4]   2> 4914175 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.e.j.s.AbstractConnector Started ServerConnector@1ec3f8d5{ssl,[ssl, alpn, http/1.1, h2]}{127.0.0.1:43087}
   [junit4]   2> 4914175 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.e.j.s.Server Started @4914201ms
   [junit4]   2> 4914175 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/zw_, solr.data.dir=hdfs://localhost.localdomain:37885/hdfs__localhost.localdomain_37885__home_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-master_checkout_solr_build_solr-core_test_J0_temp_solr.cloud.hdfs.HdfsSyncSliceTest_8F0796456E3CE4D5-001_tempDir-002_control_data, hostPort=43087, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsSyncSliceTest_8F0796456E3CE4D5-001/control-001/cores, replicaType=NRT}
   [junit4]   2> 4914175 ERROR (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 4914175 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 4914175 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 9.0.0
   [junit4]   2> 4914175 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 4914175 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 4914175 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2020-01-18T08:39:33.274796Z
   [junit4]   2> 4914177 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 4914178 INFO  (zkConnectionManagerCallback-27311-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4914178 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 4914280 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 4914280 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsSyncSliceTest_8F0796456E3CE4D5-001/control-001/solr.xml
   [junit4]   2> 4914283 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 4914283 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 4914284 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@4495597e, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 4914366 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
   [junit4]   2> 4914367 WARN  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@40a05f01[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 4914367 WARN  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@40a05f01[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 4914373 WARN  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@4caea6db[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 4914373 WARN  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@4caea6db[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 4914374 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45607/solr
   [junit4]   2> 4914376 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 4914377 INFO  (zkConnectionManagerCallback-27318-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4914377 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 4914488 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [n:127.0.0.1:43087_zw_     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 4914490 INFO  (zkConnectionManagerCallback-27320-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4914490 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [n:127.0.0.1:43087_zw_     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 4914905 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [n:127.0.0.1:43087_zw_     ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:43087_zw_
   [junit4]   2> 4914910 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [n:127.0.0.1:43087_zw_     ] o.a.s.c.Overseer Overseer (id=73090334303584261-127.0.0.1:43087_zw_-n_0000000000) starting
   [junit4]   2> 4914931 INFO  (OverseerStateUpdate-73090334303584261-127.0.0.1:43087_zw_-n_0000000000) [n:127.0.0.1:43087_zw_     ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:43087_zw_
   [junit4]   2> 4914932 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [n:127.0.0.1:43087_zw_     ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43087_zw_
   [junit4]   2> 4914937 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [n:127.0.0.1:43087_zw_     ] o.a.s.p.PackageLoader /packages.json updated to version -1
   [junit4]   2> 4914938 WARN  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [n:127.0.0.1:43087_zw_     ] o.a.s.c.CoreContainer Not all security plugins configured!  authentication=disabled authorization=disabled.  Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external.  See https://s.apache.org/solrsecurity for more info
   [junit4]   2> 4914948 INFO  (zkCallback-27319-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 4914992 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [n:127.0.0.1:43087_zw_     ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 4915463 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [n:127.0.0.1:43087_zw_     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4495597e
   [junit4]   2> 4915503 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [n:127.0.0.1:43087_zw_     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4495597e
   [junit4]   2> 4915503 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [n:127.0.0.1:43087_zw_     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4495597e
   [junit4]   2> 4915507 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [n:127.0.0.1:43087_zw_     ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsSyncSliceTest_8F0796456E3CE4D5-001/control-001/cores
   [junit4]   2> 4915524 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 4915529 INFO  (zkConnectionManagerCallback-27329-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4915529 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 4915534 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 4915545 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45607/solr ready
   [junit4]   2> 4915584 INFO  (qtp2065363905-140717) [n:127.0.0.1:43087_zw_     ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:43087_zw_&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 4915593 INFO  (OverseerThreadFactory-22361-thread-1-processing-n:127.0.0.1:43087_zw_) [n:127.0.0.1:43087_zw_     ] o.a.s.c.a.c.CreateCollectionCmd Create collection control_collection
   [junit4]   2> 4915864 INFO  (qtp2065363905-140713) [n:127.0.0.1:43087_zw_    x:control_collection_shard1_replica_n1 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 4915864 INFO  (qtp2065363905-140713) [n:127.0.0.1:43087_zw_    x:control_collection_shard1_replica_n1 ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 4916873 INFO  (qtp2065363905-140713) [n:127.0.0.1:43087_zw_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.0.0
   [junit4]   2> 4916893 INFO  (qtp2065363905-140713) [n:127.0.0.1:43087_zw_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
   [junit4]   2> 4917016 INFO  (qtp2065363905-140713) [n:127.0.0.1:43087_zw_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 4917034 INFO  (qtp2065363905-140713) [n:127.0.0.1:43087_zw_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
   [junit4]   2> 4917034 INFO  (qtp2065363905-140713) [n:127.0.0.1:43087_zw_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4495597e
   [junit4]   2> 4917034 INFO  (qtp2065363905-140713) [n:127.0.0.1:43087_zw_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost.localdomain:37885/solr_hdfs_home
   [junit4]   2> 4917034 INFO  (qtp2065363905-140713) [n:127.0.0.1:43087_zw_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
   [junit4]   2> 4917034 INFO  (qtp2065363905-140713) [n:127.0.0.1:43087_zw_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsSyncSliceTest_8F0796456E3CE4D5-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[hdfs://localhost.localdomain:37885/solr_hdfs_home/control_collection/core_node2/data/]
   [junit4]   2> 4917035 INFO  (qtp2065363905-140713) [n:127.0.0.1:43087_zw_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost.localdomain:37885/solr_hdfs_home/control_collection/core_node2/data/snapshot_metadata
   [junit4]   2> 4917041 INFO  (qtp2065363905-140713) [n:127.0.0.1:43087_zw_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
   [junit4]   2> 4917041 INFO  (qtp2065363905-140713) [n:127.0.0.1:43087_zw_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [33554432] will allocate [1] slabs and use ~[33554432] bytes
   [junit4]   2> 4917043 INFO  (qtp2065363905-140713) [n:127.0.0.1:43087_zw_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 4917044 INFO  (qtp2065363905-140713) [n:127.0.0.1:43087_zw_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost.localdomain:37885/solr_hdfs_home/control_collection/core_node2/data
   [junit4]   2> 4917056 INFO  (qtp2065363905-140713) [n:127.0.0.1:43087_zw_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost.localdomain:37885/solr_hdfs_home/control_collection/core_node2/data/index
   [junit4]   2> 4917061 INFO  (qtp2065363905-140713) [n:127.0.0.1:43087_zw_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
   [junit4]   2> 4917061 INFO  (qtp2065363905-140713) [n:127.0.0.1:43087_zw_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [33554432] will allocate [1] slabs and use ~[33554432] bytes
   [junit4]   2> 4917062 INFO  (qtp2065363905-140713) [n:127.0.0.1:43087_zw_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 4917063 INFO  (qtp2065363905-140713) [n:127.0.0.1:43087_zw_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=28, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8783730406259137]
   [junit4]   2> 4917077 WARN  (qtp2065363905-140713) [n:127.0.0.1:43087_zw_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A, b=B}}}
   [junit4]   2> 4917118 INFO  (qtp2065363905-140713) [n:127.0.0.1:43087_zw_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
   [junit4]   2> 4917118 INFO  (qtp2065363905-140713) [n:127.0.0.1:43087_zw_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 4917118 INFO  (qtp2065363905-140713) [n:127.0.0.1:43087_zw_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
   [junit4]   2> 4917126 INFO  (qtp2065363905-140713) [n:127.0.0.1:43087_zw_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 4917126 INFO  (qtp2065363905-140713) [n:127.0.0.1:43087_zw_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 4917289 INFO  (qtp2065363905-140713) [n:127.0.0.1:43087_zw_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=16, maxMergeAtOnceExplicit=18, maxMergedSegmentMB=73.048828125, floorSegmentMB=0.7978515625, forceMergeDeletesPctAllowed=10.930456264372156, segmentsPerTier=44.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0, deletesPctAllowed=20.718136085344575
   [junit4]   2> 4917309 INFO  (qtp2065363905-140713) [n:127.0.0.1:43087_zw_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.s.SolrIndexSearcher Opening [Searcher@7e84803d[control_collection_shard1_replica_n1] main]
   [junit4]   2> 4917312 INFO  (qtp2065363905-140713) [n:127.0.0.1:43087_zw_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 4917314 INFO  (qtp2065363905-140713) [n:127.0.0.1:43087_zw_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 4917314 INFO  (qtp2065363905-140713) [n:127.0.0.1:43087_zw_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 4917316 INFO  (qtp2065363905-140713) [n:127.0.0.1:43087_zw_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1656054639666135040
   [junit4]   2> 4917319 INFO  (searcherExecutor-22366-thread-1-processing-n:127.0.0.1:43087_zw_ x:control_collection_shard1_replica_n1 c:control_collection s:shard1) [n:127.0.0.1:43087_zw_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@7e84803d[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 4917331 INFO  (qtp2065363905-140713) [n:127.0.0.1:43087_zw_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.ZkShardTerms Successful update of terms at /collections/control_collection/terms/shard1 to Terms{values={core_node2=0}, version=0}
   [junit4]   2> 4917331 INFO  (qtp2065363905-140713) [n:127.0.0.1:43087_zw_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/control_collection/leaders/shard1
   [junit4]   2> 4917338 INFO  (qtp2065363905-140713) [n:127.0.0.1:43087_zw_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 4917338 INFO  (qtp2065363905-140713) [n:127.0.0.1:43087_zw_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 4917338 INFO  (qtp2065363905-140713) [n:127.0.0.1:43087_zw_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:43087/zw_/control_collection_shard1_replica_n1/
   [junit4]   2> 4917339 INFO  (qtp2065363905-140713) [n:127.0.0.1:43087_zw_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 4917339 INFO  (qtp2065363905-140713) [n:127.0.0.1:43087_zw_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.SyncStrategy https://127.0.0.1:43087/zw_/control_collection_shard1_replica_n1/ has no replicas
   [junit4]   2> 4917339 INFO  (qtp2065363905-140713) [n:127.0.0.1:43087_zw_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/control_collection/leaders/shard1/leader after winning as /collections/control_collection/leader_elect/shard1/election/73090334303584261-core_node2-n_0000000000
   [junit4]   2> 4917344 INFO  (qtp2065363905-140713) [n:127.0.0.1:43087_zw_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:43087/zw_/control_collection_shard1_replica_n1/ shard1
   [junit4]   2> 4917447 INFO  (zkCallback-27319-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 4917447 INFO  (zkCallback-27319-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 4917448 INFO  (qtp2065363905-140713) [n:127.0.0.1:43087_zw_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 4917450 INFO  (qtp2065363905-140713) [n:127.0.0.1:43087_zw_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1586
   [junit4]   2> 4917452 INFO  (qtp2065363905-140717) [n:127.0.0.1:43087_zw_     ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
   [junit4]   2> 4917550 INFO  (zkCallback-27319-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 4917550 INFO  (zkCallback-27319-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 4917550 INFO  (zkCallback-27319-thread-3) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 4917550 INFO  (qtp2065363905-140717) [n:127.0.0.1:43087_zw_     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:43087_zw_&wt=javabin&version=2} status=0 QTime=1966
   [junit4]   2> 4917566 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.AbstractFullDistribZkTestBase Waiting to see 1 active replicas in collection: control_collection
   [junit4]   2> 4917699 INFO  (OverseerCollectionConfigSetProcessor-73090334303584261-127.0.0.1:43087_zw_-n_0000000000) [n:127.0.0.1:43087_zw_     ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 4917806 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 4917806 INFO  (zkConnectionManagerCallback-27335-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4917806 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 4917809 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 4917811 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45607/solr ready
   [junit4]   2> 4917811 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 4917821 INFO  (qtp2065363905-140716) [n:127.0.0.1:43087_zw_     ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 4917825 INFO  (OverseerThreadFactory-22361-thread-2-processing-n:127.0.0.1:43087_zw_) [n:127.0.0.1:43087_zw_     ] o.a.s.c.a.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 4918030 WARN  (OverseerThreadFactory-22361-thread-2-processing-n:127.0.0.1:43087_zw_) [n:127.0.0.1:43087_zw_     ] o.a.s.c.a.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
   [junit4]   2> 4918031 INFO  (qtp2065363905-140716) [n:127.0.0.1:43087_zw_     ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
   [junit4]   2> 4918032 INFO  (qtp2065363905-140716) [n:127.0.0.1:43087_zw_     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=210
   [junit4]   2> 4918036 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.SolrCloudTestCase active slice count: 1 expected:1
   [junit4]   2> 4918036 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.SolrCloudTestCase active replica count: 0 expected replica count: 0
   [junit4]   2> 4918036 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.SolrCloudTestCase active slice count: 1 expected:1
   [junit4]   2> 4918036 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.SolrCloudTestCase active replica count: 0 expected replica count: 0
   [junit4]   2> 4918036 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.SolrCloudTestCase active slice count: 1 expected:1
   [junit4]   2> 4918036 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.SolrCloudTestCase active replica count: 0 expected replica count: 0
   [junit4]   2> 4918036 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.AbstractFullDistribZkTestBase Creating jetty instances pullReplicaCount=0 numOtherReplicas=7
   [junit4]   2> 4918577 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsSyncSliceTest_8F0796456E3CE4D5-001/shard-1-001 of type NRT
   [junit4]   2> 4918587 WARN  (closeThreadPool-27336-thread-1) [     ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
   [junit4]   2> 4918587 INFO  (closeThreadPool-27336-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 4918587 INFO  (closeThreadPool-27336-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 4918587 INFO  (closeThreadPool-27336-thread-1) [     ] o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z; git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 11.0.4+10-LTS
   [junit4]   2> 4918648 INFO  (closeThreadPool-27336-thread-1) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 4918648 INFO  (closeThreadPool-27336-thread-1) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 4918648 INFO  (closeThreadPool-27336-thread-1) [     ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 4918652 INFO  (closeThreadPool-27336-thread-1) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@505bd0eb{/zw_,null,AVAILABLE}
   [junit4]   2> 4918653 INFO  (closeThreadPool-27336-thread-1) [     ] o.e.j.s.AbstractConnector Started ServerConnector@6d7551cd{ssl,[ssl, alpn, http/1.1, h2]}{127.0.0.1:45375}
   [junit4]   2> 4918653 INFO  (closeThreadPool-27336-thread-1) [     ] o.e.j.s.Server Started @4918679ms
   [junit4]   2> 4918653 INFO  (closeThreadPool-27336-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/zw_, solrconfig=solrconfig.xml, solr.data.dir=hdfs://localhost.localdomain:37885/hdfs__localhost.localdomain_37885__home_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-master_checkout_solr_build_solr-core_test_J0_temp_solr.cloud.hdfs.HdfsSyncSliceTest_8F0796456E3CE4D5-001_tempDir-002_jetty1, hostPort=45375, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsSyncSliceTest_8F0796456E3CE4D5-001/shard-1-001/cores}
   [junit4]   2> 4918653 ERROR (closeThreadPool-27336-thread-1) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 4918653 INFO  (closeThreadPool-27336-thread-1) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 4918653 INFO  (closeThreadPool-27336-thread-1) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 9.0.0
   [junit4]   2> 4918653 INFO  (closeThreadPool-27336-thread-1) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 4918653 INFO  (closeThreadPool-27336-thread-1) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 4918653 INFO  (closeThreadPool-27336-thread-1) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2020-01-18T08:39:37.752955Z
   [junit4]   2> 4918655 INFO  (closeThreadPool-27336-thread-1) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 4918656 INFO  (zkConnectionManagerCallback-27338-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4918657 INFO  (closeThreadPool-27336-thread-1) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 4918883 INFO  (closeThreadPool-27336-thread-1) [     ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 4918883 INFO  (closeThreadPool-27336-thread-1) [     ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsSyncSliceTest_8F0796456E3CE4D5-001/shard-1-001/solr.xml
   [junit4]   2> 4918886 INFO  (closeThreadPool-27336-thread-1) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 4918886 INFO  (closeThreadPool-27336-thread-1) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 4918888 INFO  (closeThreadPool-27336-thread-1) [     ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@4495597e, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 4918995 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsSyncSliceTest_8F0796456E3CE4D5-001/shard-2-001 of type NRT
   [junit4]   2> 4919011 WARN  (closeThreadPool-27336-thread-2) [     ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
   [junit4]   2> 4919011 INFO  (closeThreadPool-27336-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 4919011 INFO  (closeThreadPool-27336-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 4919011 INFO  (closeThreadPool-27336-thread-2) [     ] o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z; git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 11.0.4+10-LTS
   [junit4]   2> 4919266 INFO  (closeThreadPool-27336-thread-2) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 4919266 INFO  (closeThreadPool-27336-thread-2) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 4919266 INFO  (closeThreadPool-27336-thread-2) [     ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 4919266 INFO  (closeThreadPool-27336-thread-2) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3be3b30d{/zw_,null,AVAILABLE}
   [junit4]   2> 4919267 INFO  (closeThreadPool-27336-thread-2) [     ] o.e.j.s.AbstractConnector Started ServerConnector@4957e837{ssl,[ssl, alpn, http/1.1, h2]}{127.0.0.1:42611}
   [junit4]   2> 4919267 INFO  (closeThreadPool-27336-thread-2) [     ] o.e.j.s.Server Started @4919293ms
   [junit4]   2> 4919267 INFO  (closeThreadPool-27336-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/zw_, solrconfig=solrconfig.xml, solr.data.dir=hdfs://localhost.localdomain:37885/hdfs__localhost.localdomain_37885__home_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-master_checkout_solr_build_solr-core_test_J0_temp_solr.cloud.hdfs.HdfsSyncSliceTest_8F0796456E3CE4D5-001_tempDir-002_jetty2, hostPort=42611, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsSyncSliceTest_8F0796456E3CE4D5-001/shard-2-001/cores}
   [junit4]   2> 4919267 ERROR (closeThreadPool-27336-thread-2) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 4919267 INFO  (closeThreadPool-27336-thread-2) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 4919267 INFO  (closeThreadPool-27336-thread-2) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 9.0.0
   [junit4]   2> 4919267 INFO  (closeThreadPool-27336-thread-2) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 4919267 INFO  (closeThreadPool-27336-thread-2) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 4919267 INFO  (closeThreadPool-27336-thread-2) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2020-01-18T08:39:38.366316Z
   [junit4]   2> 4919268 INFO  (closeThreadPool-27336-thread-2) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 4919270 INFO  (zkConnectionManagerCallback-27341-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4919270 INFO  (closeThreadPool-27336-thread-2) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 4919373 INFO  (closeThreadPool-27336-thread-2) [     ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 4919373 INFO  (closeThreadPool-27336-thread-2) [     ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsSyncSliceTest_8F0796456E3CE4D5-001/shard-2-001/solr.xml
   [junit4]   2> 4919375 INFO  (closeThreadPool-27336-thread-2) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 4919375 INFO  (closeThreadPool-27336-thread-2) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 4919377 INFO  (closeThreadPool-27336-thread-2) [     ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@4495597e, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 4919430 INFO  (closeThreadPool-27336-thread-1) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
   [junit4]   2> 4919431 WARN  (closeThreadPool-27336-thread-1) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@1aa3cf78[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 4919431 WARN  (closeThreadPool-27336-thread-1) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@1aa3cf78[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 4919437 WARN  (closeThreadPool-27336-thread-1) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@540ce959[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 4919437 WARN  (closeThreadPool-27336-thread-1) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@540ce959[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 4919439 INFO  (closeThreadPool-27336-thread-1) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45607/solr
   [junit4]   2> 4919449 INFO  (closeThreadPool-27336-thread-1) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 4919456 INFO  (zkConnectionManagerCallback-27348-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4919456 INFO  (closeThreadPool-27336-thread-1) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 4919568 INFO  (closeThreadPool-27336-thread-1) [n:127.0.0.1:45375_zw_     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 4919574 INFO  (closeThreadPool-27336-thread-2) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
   [junit4]   2> 4919574 WARN  (closeThreadPool-27336-thread-2) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@234d787e[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 4919574 WARN  (closeThreadPool-27336-thread-2) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@234d787e[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 4919575 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsSyncSliceTest_8F0796456E3CE4D5-001/shard-3-001 of type NRT
   [junit4]   2> 4919578 WARN  (closeThreadPool-27336-thread-3) [     ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
   [junit4]   2> 4919578 INFO  (closeThreadPool-27336-thread-3) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 4919578 INFO  (closeThreadPool-27336-thread-3) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 4919578 INFO  (closeThreadPool-27336-thread-3) [     ] o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z; git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 11.0.4+10-LTS
   [junit4]   2> 4919646 INFO  (zkConnectionManagerCallback-27350-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4919727 INFO  (closeThreadPool-27336-thread-1) [n:127.0.0.1:45375_zw_     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 4919728 INFO  (closeThreadPool-27336-thread-3) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 4919728 INFO  (closeThreadPool-27336-thread-3) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 4919728 INFO  (closeThreadPool-27336-thread-3) [     ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 4919729 INFO  (closeThreadPool-27336-thread-3) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@183df39a{/zw_,null,AVAILABLE}
   [junit4]   2> 4919730 WARN  (closeThreadPool-27336-thread-2) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@18cae68e[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 4919730 WARN  (closeThreadPool-27336-thread-2) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@18cae68e[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 4919730 INFO  (closeThreadPool-27336-thread-3) [     ] o.e.j.s.AbstractConnector Started ServerConnector@701bce9b{ssl,[ssl, alpn, http/1.1, h2]}{127.0.0.1:36251}
   [junit4]   2> 4919730 INFO  (closeThreadPool-27336-thread-3) [     ] o.e.j.s.Server Started @4919757ms
   [junit4]   2> 4919730 INFO  (closeThreadPool-27336-thread-3) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/zw_, solrconfig=solrconfig.xml, solr.data.dir=hdfs://localhost.localdomain:37885/hdfs__localhost.localdomain_37885__home_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-master_checkout_solr_build_solr-core_test_J0_temp_solr.cloud.hdfs.HdfsSyncSliceTest_8F0796456E3CE4D5-001_tempDir-002_jetty3, hostPort=36251, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsSyncSliceTest_8F0796456E3CE4D5-001/shard-3-001/cores}
   [junit4]   2> 4919731 ERROR (closeThreadPool-27336-thread-3) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 4919731 INFO  (closeThreadPool-27336-thread-3) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 4919731 INFO  (closeThreadPool-27336-thread-3) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 9.0.0
   [junit4]   2> 4919731 INFO  (closeThreadPool-27336-thread-3) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 4919731 INFO  (closeThreadPool-27336-thread-3) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 4919731 INFO  (closeThreadPool-27336-thread-3) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2020-01-18T08:39:38.830104Z
   [junit4]   2> 4919732 INFO  (closeThreadPool-27336-thread-2) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45607/solr
   [junit4]   2> 4919734 INFO  (closeThreadPool-27336-thread-3) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 4919735 INFO  (closeThreadPool-27336-thread-1) [n:127.0.0.1:45375_zw_     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 4919735 INFO  (closeThreadPool-27336-thread-2) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 4919736 INFO  (zkConnectionManagerCallback-27358-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4919736 INFO  (closeThreadPool-27336-thread-3) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 4919738 INFO  (zkConnectionManagerCallback-27360-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4919738 INFO  (closeThreadPool-27336-thread-2) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 4919742 INFO  (closeThreadPool-27336-thread-1) [n:127.0.0.1:45375_zw_     ] o.a.s.c.ZkController Publish node=127.0.0.1:45375_zw_ as DOWN
   [junit4]   2> 4919744 INFO  (closeThreadPool-27336-thread-1) [n:127.0.0.1:45375_zw_     ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 4919744 INFO  (closeThreadPool-27336-thread-1) [n:127.0.0.1:45375_zw_     ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45375_zw_
   [junit4]   2> 4919745 INFO  (zkCallback-27334-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 4919745 INFO  (zkCallback-27319-thread-3) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 4919745 INFO  (zkCallback-27349-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 4919748 INFO  (closeThreadPool-27336-thread-1) [n:127.0.0.1:45375_zw_     ] o.a.s.p.PackageLoader /packages.json updated to version -1
   [junit4]   2> 4919748 WARN  (closeThreadPool-27336-thread-1) [n:127.0.0.1:45375_zw_     ] o.a.s.c.CoreContainer Not all security plugins configured!  authentication=disabled authorization=disabled.  Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external.  See https://s.apache.org/solrsecurity for more info
   [junit4]   2> 4919767 INFO  (closeThreadPool-27336-thread-1) [n:127.0.0.1:45375_zw_     ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 4919826 INFO  (OverseerCollectionConfigSetProcessor-73090334303584261-127.0.0.1:43087_zw_-n_0000000000) [n:127.0.0.1:43087_zw_     ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 4919831 INFO  (closeThreadPool-27336-thread-1) [n:127.0.0.1:45375_zw_     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4495597e
   [junit4]   2> 4919847 INFO  (closeThreadPool-27336-thread-1) [n:127.0.0.1:45375_zw_     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4495597e
   [junit4]   2> 4919847 INFO  (closeThreadPool-27336-thread-1) [n:127.0.0.1:45375_zw_     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4495597e
   [junit4]   2> 4919849 INFO  (closeThreadPool-27336-thread-1) [n:127.0.0.1:45375_zw_     ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsSyncSliceTest_8F0796456E3CE4D5-001/shard-1-001/cores
   [junit4]   2> 4919858 INFO  (closeThreadPool-27336-thread-2) [n:127.0.0.1:42611_zw_     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 4919865 INFO  (zkConnectionManagerCallback-27364-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4919865 INFO  (closeThreadPool-27336-thread-2) [n:127.0.0.1:42611_zw_     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 4919882 INFO  (closeThreadPool-27336-thread-2) [n:127.0.0.1:42611_zw_     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 4919892 INFO  (closeThreadPool-27336-thread-2) [n:127.0.0.1:42611_zw_     ] o.a.s.c.ZkController Publish node=127.0.0.1:42611_zw_ as DOWN
   [junit4]   2> 4919893 INFO  (closeThreadPool-27336-thread-2) [n:127.0.0.1:42611_zw_     ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 4919893 INFO  (closeThreadPool-27336-thread-2) [n:127.0.0.1:42611_zw_     ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:42611_zw_
   [junit4]   2> 4919896 INFO  (zkCallback-27319-thread-3) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 4919897 INFO  (zkCallback-27349-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 4919897 INFO  (zkCallback-27334-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 4919900 INFO  (closeThreadPool-27336-thread-2) [n:127.0.0.1:42611_zw_     ] o.a.s.p.PackageLoader /packages.json updated to version -1
   [junit4]   2> 4919900 WARN  (closeThreadPool-27336-thread-2) [n:127.0.0.1:42611_zw_     ] o.a.s.c.CoreContainer Not all security plugins configured!  authentication=disabled authorization=disabled.  Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external.  See https://s.apache.org/solrsecurity for more info
   [junit4]   2> 4919904 INFO  (zkCallback-27363-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 4919907 INFO  (closeThreadPool-27336-thread-1) [     ] o.a.s.c.AbstractFullDistribZkTestBase waitForLiveNode: 127.0.0.1:45375_zw_
   [junit4]   2> 4919936 INFO  (closeThreadPool-27336-thread-2) [n:127.0.0.1:42611_zw_     ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 4919939 INFO  (closeThreadPool-27336-thread-3) [     ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 4919939 INFO  (closeThreadPool-27336-thread-3) [     ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsSyncSliceTest_8F0796456E3CE4D5-001/shard-3-001/solr.xml
   [junit4]   2> 4919942 INFO  (closeThreadPool-27336-thread-3) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 4919942 INFO  (closeThreadPool-27336-thread-3) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 4919943 INFO  (closeThreadPool-27336-thread-3) [     ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@4495597e, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 4919962 INFO  (closeThreadPool-27336-thread-2) [n:127.0.0.1:42611_zw_     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4495597e
   [junit4]   2> 4920047 INFO  (closeThreadPool-27336-thread-2) [n:127.0.0.1:42611_zw_     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4495597e
   [junit4]   2> 4920048 INFO  (closeThreadPool-27336-thread-2) [n:127.0.0.1:42611_zw_     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4495597e
   [junit4]   2> 4920049 INFO  (closeThreadPool-27336-thread-2) [n:127.0.0.1:42611_zw_     ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsSyncSliceTest_8F0796456E3CE4D5-001/shard-2-001/cores
   [junit4]   2> 4920448 INFO  (closeThreadPool-27336-thread-2) [     ] o.a.s.c.AbstractFullDistribZkTestBase waitForLiveNode: 127.0.0.1:42611_zw_
   [junit4]   2> 4920515 INFO  (closeThreadPool-27336-thread-3) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
   [junit4]   2> 4920527 WARN  (closeThreadPool-27336-thread-3) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@a77acab[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 4920528 WARN  (closeThreadPool-27336-thread-3) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@a77acab[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 4920663 WARN  (closeThreadPool-27336-thread-3) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@5b24937e[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 4920663 WARN  (closeThreadPool-27336-thread-3) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@5b24937e[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 4920665 INFO  (closeThreadPool-27336-thread-3) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45607/solr
   [junit4]   2> 4920666 INFO  (closeThreadPool-27336-thread-3) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 4920671 INFO  (zkConnectionManagerCallback-27375-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4920671 INFO  (closeThreadPool-27336-thread-3) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 4920676 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 4 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsSyncSliceTest_8F0796456E3CE4D5-001/shard-4-001 of type NRT
   [junit4]   2> 4920678 WARN  (closeThreadPool-27336-thread-2) [     ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
   [junit4]   2> 4920678 INFO  (closeThreadPool-27336-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 4920678 INFO  (closeThreadPool-27336-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 4920678 INFO  (closeThreadPool-27336-thread-2) [     ] o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z; git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 11.0.4+10-LTS
   [junit4]   2> 4920680 INFO  (closeThreadPool-27336-thread-2) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 4920680 INFO  (closeThreadPool-27336-thread-2) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 4920680 INFO  (closeThreadPool-27336-thread-2) [     ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 4920681 INFO  (closeThreadPool-27336-thread-2) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6a9d04dc{/zw_,null,AVAILABLE}
   [junit4]   2> 4920681 INFO  (closeThreadPool-27336-thread-2) [     ] o.e.j.s.AbstractConnector Started ServerConnector@245c1b5e{ssl,[ssl, alpn, http/1.1, h2]}{127.0.0.1:44501}
   [junit4]   2> 4920681 INFO  (closeThreadPool-27336-thread-2) [     ] o.e.j.s.Server Started @4920708ms
   [junit4]   2> 4920681 INFO  (closeThreadPool-27336-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/zw_, solrconfig=solrconfig.xml, solr.data.dir=hdfs://localhost.localdomain:37885/hdfs__localhost.localdomain_37885__home_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-master_checkout_solr_build_solr-core_test_J0_temp_solr.cloud.hdfs.HdfsSyncSliceTest_8F0796456E3CE4D5-001_tempDir-002_jetty4, hostPort=44501, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsSyncSliceTest_8F0796456E3CE4D5-001/shard-4-001/cores, replicaType=NRT}
   [junit4]   2> 4920682 ERROR (closeThreadPool-27336-thread-2) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 4920682 INFO  (closeThreadPool-27336-thread-2) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 4920682 INFO  (closeThreadPool-27336-thread-2) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 9.0.0
   [junit4]   2> 4920682 INFO  (closeThreadPool-27336-thread-2) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 4920682 INFO  (closeThreadPool-27336-thread-2) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 4920682 INFO  (closeThreadPool-27336-thread-2) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2020-01-18T08:39:39.781153Z
   [junit4]   2> 4920683 INFO  (closeThreadPool-27336-thread-2) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 4920685 INFO  (zkConnectionManagerCallback-27377-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4920685 INFO  (closeThreadPool-27336-thread-2) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 4920774 INFO  (closeThreadPool-27336-thread-3) [n:127.0.0.1:36251_zw_     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 4920777 INFO  (zkConnectionManagerCallback-27379-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4920777 INFO  (closeThreadPool-27336-thread-3) [n:127.0.0.1:36251_zw_     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 4920785 INFO  (closeThreadPool-27336-thread-3) [n:127.0.0.1:36251_zw_     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 4920793 INFO  (closeThreadPool-27336-thread-3) [n:127.0.0.1:36251_zw_     ] o.a.s.c.ZkController Publish node=127.0.0.1:36251_zw_ as DOWN
   [junit4]   2> 4920794 INFO  (closeThreadPool-27336-thread-3) [n:127.0.0.1:36251_zw_     ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 4920794 INFO  (closeThreadPool-27336-thread-3) [n:127.0.0.1:36251_zw_     ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:36251_zw_
   [junit4]   2> 4920808 INFO  (zkCallback-27334-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 4920808 INFO  (zkCallback-27349-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 4920808 INFO  (zkCallback-27363-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 4920808 INFO  (zkCallback-27319-thread-3) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 4920810 INFO  (zkCallback-27378-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 4920813 INFO  (closeThreadPool-27336-thread-3) [n:127.0.0.1:36251_zw_     ] o.a.s.p.PackageLoader /packages.json updated to version -1
   [junit4]   2> 4920813 WARN  (closeThreadPool-27336-thread-3) [n:127.0.0.1:36251_zw_     ] o.a.s.c.CoreContainer Not all security plugins configured!  authentication=disabled authorization=disabled.  Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external.  See https://s.apache.org/solrsecurity for more info
   [junit4]   2> 4920830 INFO  (closeThreadPool-27336-thread-3) [n:127.0.0.1:36251_zw_     ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 4920844 INFO  (closeThreadPool-27336-thread-3) [n:127.0.0.1:36251_zw_     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4495597e
   [junit4]   2> 4920851 INFO  (closeThreadPool-27336-thread-3) [n:127.0.0.1:36251_zw_     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4495597e
   [junit4]   2> 4920851 INFO  (closeThreadPool-27336-thread-3) [n:127.0.0.1:36251_zw_     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4495597e
   [junit4]   2> 4920852 INFO  (closeThreadPool-27336-thread-3) [n:127.0.0.1:36251_zw_     ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsSyncSliceTest_8F0796456E3CE4D5-001/shard-3-001/cores
   [junit4]   2> 4920874 INFO  (closeThreadPool-27336-thread-2) [     ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 4920874 INFO  (closeThreadPool-27336-thread-2) [     ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsSyncSliceTest_8F0796456E3CE4D5-001/shard-4-001/solr.xml
   [junit4]   2> 4920876 INFO  (closeThreadPool-27336-thread-2) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 4920876 INFO  (closeThreadPool-27336-thread-2) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 4920877 INFO  (closeThreadPool-27336-thread-2) [     ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@4495597e, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 4920879 INFO  (closeThreadPool-27336-thread-3) [     ] o.a.s.c.AbstractFullDistribZkTestBase waitForLiveNode: 127.0.0.1:36251_zw_
   [junit4]   2> 4921111 INFO  (TEST-HdfsSyncSliceTest.test-seed#[8F0796456E3CE4D5]) [     ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 5 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsSyncSliceTest_8F0796456E3CE4D5-001/shard-5-001 of type NRT
   [junit4]   2> 4921113 WARN  (closeThreadPool-27336-thread-3) [     ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
   [junit4]   2> 4921113 INFO  (closeThreadPool-27336-thread-3) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 4921113 INFO  (closeThreadPool-27336-thread-3) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 4921113 INFO  (closeThreadPool-27336-thread-3) [     ] o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z; git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 11.0.4+10-LTS
   [junit4]   2> 4921123 INFO  (closeThreadPool-27336-thread-3) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 4921123 INFO  (closeThreadPool-27336-thread-3) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 4921123 INFO  (closeThreadPool-27336-thread-3) [     ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 4921123 INFO  (closeThreadPool-27336-thread-3) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@25bc0614{/zw_,null,AVAILABLE}
   [junit4]   2> 4921125 INFO  (closeThreadPool-27336-thread-3) [     ] o.e.j.s.AbstractConnector Started ServerConnector@259d0a8c{ssl,[ssl, alpn, http/1.1, h2]}{127.0.0.1:35599}
   [junit4]   2> 4921125 INFO  (closeThreadPool-27336-thread-3) [     ] o.e.j.s.Server Started @4921151ms
   [junit4]   2> 4921125 INFO  (closeThreadPool-27336-thread-3) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/zw_, solrconfig=solrconfig.xml, solr.data.dir=hdfs://localhost.localdomain:37885/hdfs__localhost.localdomain_37885__home_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-master_checkout_solr_build_solr-core_test_J0_temp_solr.cloud.hdfs.HdfsSyncSliceTest_8F0796456E3CE4D5-001_tempDir-002_jetty5, hostPort=35599, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsSyncSliceTest_8F0796456E3CE4D5-001/shard-5-001/cores, replicaType=NRT}
   [junit4]   2> 4921125 ERROR (closeThreadPool-27336-thread-3) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 4921125 INFO  (closeThreadPool-27336-thread-3) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 4921125 INFO  (closeThreadPool-27336-thread-3) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 9.0.0
   [junit4]   2> 4921125 INFO  (closeThreadPool-27336-thread-3) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 4921125 INFO  (closeThreadPool-27336-thread-3) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 4921125 INFO  (closeThreadPool-27336-t

[...truncated too long message...]

D5]-worker) [     ] o.e.j.s.AbstractConnector Stopped ServerConnector@6476d43b{HTTP/1.1,[http/1.1]}{localhost:0}
   [junit4]   2> 4952388 INFO  (SUITE-HdfsSyncSliceTest-seed#[8F0796456E3CE4D5]-worker) [     ] o.e.j.s.session node0 Stopped scavenging
   [junit4]   2> 4952388 INFO  (SUITE-HdfsSyncSliceTest-seed#[8F0796456E3CE4D5]-worker) [     ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@1ed8f9da{static,/static,jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/static,UNAVAILABLE}
   [junit4]   2> 4952389 WARN  (BP-1509085976-127.0.0.1-1579336770771 heartbeating to localhost.localdomain/127.0.0.1:37885) [     ] o.a.h.h.s.d.IncrementalBlockReportManager IncrementalBlockReportManager interrupted
   [junit4]   2> 4952389 WARN  (BP-1509085976-127.0.0.1-1579336770771 heartbeating to localhost.localdomain/127.0.0.1:37885) [     ] o.a.h.h.s.d.DataNode Ending block pool service for: Block pool BP-1509085976-127.0.0.1-1579336770771 (Datanode Uuid 33ac210d-2c91-4cc6-90cd-76290ee12992) service to localhost.localdomain/127.0.0.1:37885
   [junit4]   2> 4952406 INFO  (SUITE-HdfsSyncSliceTest-seed#[8F0796456E3CE4D5]-worker) [     ] o.e.j.s.h.ContextHandler Stopped o.e.j.w.WebAppContext@73233ef6{hdfs,/,null,UNAVAILABLE}{jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/hdfs}
   [junit4]   2> 4952406 INFO  (SUITE-HdfsSyncSliceTest-seed#[8F0796456E3CE4D5]-worker) [     ] o.e.j.s.AbstractConnector Stopped ServerConnector@4e8cb4d{HTTP/1.1,[http/1.1]}{localhost.localdomain:0}
   [junit4]   2> 4952406 INFO  (SUITE-HdfsSyncSliceTest-seed#[8F0796456E3CE4D5]-worker) [     ] o.e.j.s.session node0 Stopped scavenging
   [junit4]   2> 4952406 INFO  (SUITE-HdfsSyncSliceTest-seed#[8F0796456E3CE4D5]-worker) [     ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@532ff958{static,/static,jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/static,UNAVAILABLE}
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsSyncSliceTest_8F0796456E3CE4D5-001
   [junit4]   2> Jan 18, 2020 8:40:11 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 33 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Lucene84, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@40d57f5e), locale=ksb-TZ, timezone=Africa/Bamako
   [junit4]   2> NOTE: Linux 4.15.0-54-generic amd64/Oracle Corporation 11.0.4 (64-bit)/cpus=4,threads=4,free=262133008,total=536870912
   [junit4]   2> NOTE: All tests run in this JVM: [TestBM25SimilarityFactory, XsltUpdateRequestHandlerTest, TestStressCloudBlindAtomicUpdates, TestLegacyTerms, RestartWhileUpdatingTest, PeerSyncWithLeaderAndIndexFingerprintCachingTest, TestFuzzyAnalyzedSuggestions, DirectSolrConnectionTest, TestHttpShardHandlerFactory, TestMiniSolrCloudClusterSSL, DistributedFacetPivotSmallTest, TestXIncludeConfig, HdfsBasicDistributedZk2Test, MetricTriggerTest, TestInPlaceUpdatesDistrib, MissingSegmentRecoveryTest, TestSolrCloudWithDelegationTokens, TestManagedSchemaThreadSafety, TestConfigSetProperties, TestSimDistributedQueue, TestDynamicFieldCollectionResource, SolrCLIZkUtilsTest, ScheduledTriggerTest, SuggesterWFSTTest, TestTrie, UtilsToolTest, HdfsCollectionsAPIDistributedZkTest, RequestLoggingTest, TestReplicationHandlerDiskOverFlow, AnalyticsQueryTest, TestWithCollection, OverriddenZkACLAndCredentialsProvidersTest, TestInitParams, AtomicUpdateRemovalJavabinTest, MetricsHistoryIntegrationTest, TestCharFilters, SOLR749Test, TestCloudNestedDocsSort, TestSolrCLIRunExample, TestSimTriggerIntegration, TestBlendedInfixSuggestions, TestPullReplica, HighlighterConfigTest, SplitHandlerTest, TestJavabinTupleStreamParser, TestDFRSimilarityFactory, TestCollapseQParserPlugin, TestFaceting, TestNoOpRegenerator, SampleTest, TestElisionMultitermQuery, EchoParamsTest, CoreAdminCreateDiscoverTest, HdfsDirectoryTest, AnalysisErrorHandlingTest, HdfsWriteToMultipleCollectionsTest, TestManagedStopFilterFactory, IndexSizeTriggerSizeEstimationTest, SplitShardTest, TestGraphTermsQParserPlugin, TestQueryUtils, TestSimClusterStateProvider, TestRangeQuery, TestSortableTextField, TestSimGenericDistributedQueue, TestFieldResource, TermsComponentTest, CdcrVersionReplicationTest, ClassificationUpdateProcessorFactoryTest, CollectionsAPISolrJTest, BasicZkTest, TestCoreDiscovery, TestDynamicURP, SegmentsInfoRequestHandlerTest, RandomizedTaggerTest, ActionThrottleTest, TestCrossCoreJoin, TestQuerySenderNoQuery, SimpleFacetsTest, TestShortCircuitedRequests, OrderedExecutorTest, TestScoreJoinQPScore, TestPostingsSolrHighlighter, TestStressRecovery, TestHdfsCloudBackupRestore, TestLMJelinekMercerSimilarityFactory, TestGroupingSearch, UpdateRequestProcessorFactoryTest, PhrasesIdentificationComponentTest, TestExportTool, TestFieldCacheSort, TestSolrCloudWithHadoopAuthPlugin, ResponseLogComponentTest, FuzzySearchTest, TestTrieFacet, TestSimpleQParserPlugin, ClusterStateUpdateTest, SpatialHeatmapFacetsTest, ForceLeaderTest, SplitByPrefixTest, HealthCheckHandlerTest, SpellCheckCollatorTest, TestSolrQueryResponse, ExplicitHLLTest, TestSimNodeLostTrigger, CoreAdminRequestStatusTest, TestAuthenticationFramework, TestAuthorizationFramework, TestPayloadScoreQParserPlugin, RAMDirectoryFactoryTest, DistributedQueryComponentCustomSortTest, TestRawResponseWriter, TestCloudPivotFacet, MetricsHistoryHandlerTest, TestNumericTerms64, TestJsonFacetsWithNestedObjects, TestSolrDeletionPolicy2, CoreSorterTest, CleanupOldIndexTest, MetricUtilsTest, DocValuesMultiTest, URLClassifyProcessorTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, TestConfigSetsAPI, BasicAuthOnSingleNodeTest, AnalysisAfterCoreReloadTest, AlternateDirectoryTest, TestSchemaNameResource, TestRecovery, TestEmbeddedSolrServerSchemaAPI, FacetPivotSmallTest, TestPerFieldSimilarity, DirectUpdateHandlerTest, TestSolr4Spatial2, TestStreamBody, HdfsThreadLeakTest, UninvertDocValuesMergePolicyTest, TestSolrCoreSnapshots, DistributedQueueTest, LeaderTragicEventTest, TestDistributedMap, DistributedDebugComponentTest, TestBinaryField, SoftAutoCommitTest, TestHalfAndHalfDocValues, BigEndianAscendingWordDeserializerTest, CursorPagingTest, DisMaxRequestHandlerTest, DistributedIntervalFacetingTest, TestDistributedMissingSort, TestDistributedSearch, TestDocumentBuilder, TestRandomDVFaceting, TestRandomFaceting, TestSimpleTrackingShardHandler, TestSolrCoreProperties, TestSolrTestCaseJ4, TestTolerantSearch, TestReversedWildcardFilterFactory, TestWordDelimiterFilterFactory, TokenizerChainTest, TestLuceneIndexBackCompat, TestEmbeddedSolrServerAdminHandler, TestEmbeddedSolrServerConstructors, AssignBackwardCompatibilityTest, BasicDistributedZk2Test, BasicDistributedZkTest, ChaosMonkeySafeLeaderTest, ChaosMonkeySafeLeaderWithPullReplicasTest, ConnectionManagerTest, CreateCollectionCleanupTest, CreateRoutedAliasTest, DeleteInactiveReplicaTest, DeleteLastCustomShardedReplicaTest, DeleteNodeTest, DeleteReplicaTest, DeleteShardTest, DeleteStatusTest, DistribCursorPagingTest, DistribDocExpirationUpdateProcessorTest, DistribJoinFromCollectionTest, HttpPartitionWithTlogReplicasTest, LeaderElectionContextKeyTest, LeaderElectionIntegrationTest, LeaderElectionTest, LeaderFailoverAfterPartitionTest, LeaderFailureAfterFreshStartTest, MoveReplicaTest, MultiSolrCloudTestCaseTest, MultiThreadedOCPTest, NestedShardedAtomicUpdateTest, NodeMutatorTest, OutOfBoxZkACLAndCredentialsProvidersTest, OverseerRolesTest, OverseerSolrResponseTest, OverseerSolrResponseUnsafeSerializationTest, OverseerStatusTest, OverseerTaskQueueTest, OverseerTest, PackageManagerCLITest, PeerSyncReplicationTest, RecoveryAfterSoftCommitTest, RecoveryZkTest, ReindexCollectionTest, RemoteQueryErrorTest, ReplaceNodeNoTargetTest, ReplaceNodeTest, ReplicationFactorTest, SSLMigrationTest, SaslZkACLProviderTest, ShardRoutingCustomTest, ShardRoutingTest, SharedFSAutoReplicaFailoverTest, SliceStateTest, TestCloudDeleteByQuery, TestCloudInspectUtil, TestCloudPhrasesIdentificationComponent, TestCloudRecovery2, TestCloudSearcherWarming, TestClusterProperties, TestCryptoKeys, TestDeleteCollectionOnDownNodes, TestDistribDocBasedVersion, TestExactSharedStatsCacheCloud, TestExactStatsCacheCloud, TestExclusionRuleCollectionAccess, TestHashPartitioner, TestLRUStatsCacheCloud, TestLeaderElectionWithEmptyReplica, TestLeaderElectionZkExpiry, TestLocalStatsCacheCloud, TestLockTree, TestRandomFlRTGCloud, TestRandomRequestDistribution, TestRebalanceLeaders, TestRequestForwarding, TestSSLRandomization, TestSegmentSorting, TestTlogReplayVsRecovery, TestTlogReplica, TestTolerantUpdateProcessorCloud, TestTolerantUpdateProcessorRandomCloud, TestUtilizeNode, TestWaitForStateWithJettyShutdowns, TriLevelCompositeIdRoutingTest, UnloadDistributedZkTest, VMParamsZkACLAndCredentialsProvidersTest, ZkCLITest, ZkControllerTest, ZkFailoverTest, ZkNodePropsTest, ZkShardTermsTest, ZkSolrClientTest, AssignTest, AsyncCallRequestStatusResponseTest, CollectionReloadTest, CollectionTooManyReplicasTest, CollectionsAPIAsyncDistributedZkTest, CollectionsAPIDistributedZkTest, TestRequestStatusCollectionAPI, AutoAddReplicasPlanActionTest, AutoScalingHandlerTest, ComputePlanActionTest, HttpTriggerListenerTest, IndexSizeTriggerMixedBoundsTest, ScheduledTriggerIntegrationTest, SystemLogListenerTest, TriggerCooldownIntegrationTest, TriggerSetPropertiesIntegrationTest, BaseCdcrDistributedZkTest, CdcrBidirectionalTest, CdcrBootstrapTest, CdcrReplicationHandlerTest, CdcrRequestHandlerTest, HdfsChaosMonkeySafeLeaderTest, HdfsNNFailoverTest, HdfsRecoveryZkTest, HdfsSyncSliceTest]
   [junit4] Completed [595/900 (1!)] on J0 in 43.92s, 1 test, 1 failure <<< FAILURES!

[...truncated 53488 lines...]

[JENKINS] Lucene-Solr-NightlyTests-master - Build # 2078 - Still Unstable

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/2078/

1 tests failed.
FAILED:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test

Error Message:
document count mismatch.  control=295 sum(shards)=0 cloudClient=0

Stack Trace:
java.lang.AssertionError: document count mismatch.  control=295 sum(shards)=0 cloudClient=0
	at __randomizedtesting.SeedInfo.seed([BCECCDC0F80649F5:34B8F21A56FA240D]:0)
	at org.junit.Assert.fail(Assert.java:88)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1522)
	at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:266)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1082)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1054)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:834)




Build Log:
[...truncated 14096 lines...]
   [junit4] Suite: org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest
   [junit4]   2> 1931344 INFO  (SUITE-ChaosMonkeyNothingIsSafeTest-seed#[BCECCDC0F80649F5]-worker) [     ] o.a.s.SolrTestCaseJ4 Created dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_BCECCDC0F80649F5-001/data-dir-83-001
   [junit4]   2> 1931344 WARN  (SUITE-ChaosMonkeyNothingIsSafeTest-seed#[BCECCDC0F80649F5]-worker) [     ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=102 numCloses=102
   [junit4]   2> 1931344 INFO  (SUITE-ChaosMonkeyNothingIsSafeTest-seed#[BCECCDC0F80649F5]-worker) [     ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 1931345 INFO  (SUITE-ChaosMonkeyNothingIsSafeTest-seed#[BCECCDC0F80649F5]-worker) [     ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl="https://issues.apache.org/jira/browse/SOLR-5776")
   [junit4]   2> 1931345 INFO  (SUITE-ChaosMonkeyNothingIsSafeTest-seed#[BCECCDC0F80649F5]-worker) [     ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 1931345 INFO  (SUITE-ChaosMonkeyNothingIsSafeTest-seed#[BCECCDC0F80649F5]-worker) [     ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /c_dcv/
   [junit4]   2> 1931348 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1931349 INFO  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1931349 INFO  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1931449 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.ZkTestServer start zk server on port:45638
   [junit4]   2> 1931449 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.ZkTestServer waitForServerUp: 127.0.0.1:45638
   [junit4]   2> 1931449 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:45638
   [junit4]   2> 1931449 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 45638
   [junit4]   2> 1931451 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1931452 INFO  (zkConnectionManagerCallback-9451-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1931452 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1931454 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1931455 INFO  (zkConnectionManagerCallback-9453-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1931455 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1931457 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1931458 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 1931460 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1931462 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1931471 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1931472 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1931473 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1931474 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1931475 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1931476 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1931477 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1931478 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
   [junit4]   2> 1931748 WARN  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
   [junit4]   2> 1931748 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 1931748 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 1931748 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z; git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 11.0.4+10-LTS
   [junit4]   2> 1931749 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1931749 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1931749 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 1931750 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@791e8298{/c_dcv,null,AVAILABLE}
   [junit4]   2> 1931750 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.e.j.s.AbstractConnector Started ServerConnector@82cfb3e{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:41201}
   [junit4]   2> 1931750 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.e.j.s.Server Started @1931791ms
   [junit4]   2> 1931750 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/c_dcv, solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_BCECCDC0F80649F5-001/tempDir-001/control/data, hostPort=41201, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_BCECCDC0F80649F5-001/control-001/cores}
   [junit4]   2> 1931751 ERROR (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1931751 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 1931751 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 9.0.0
   [junit4]   2> 1931751 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1931751 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1931751 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2020-01-19T08:07:54.883231Z
   [junit4]   2> 1931752 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1931753 INFO  (zkConnectionManagerCallback-9455-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1931753 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1931854 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 1931854 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_BCECCDC0F80649F5-001/control-001/solr.xml
   [junit4]   2> 1931856 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 1931856 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 1931858 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@67d3da70, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 1932078 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
   [junit4]   2> 1932079 WARN  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@43d8ebe3[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1932079 WARN  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@43d8ebe3[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1932081 WARN  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@7c33fa47[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1932081 WARN  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@7c33fa47[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1932082 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45638/solr
   [junit4]   2> 1932083 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1932084 INFO  (zkConnectionManagerCallback-9462-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1932084 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1932186 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [n:127.0.0.1:41201_c_dcv     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1932187 INFO  (zkConnectionManagerCallback-9464-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1932187 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [n:127.0.0.1:41201_c_dcv     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1932291 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [n:127.0.0.1:41201_c_dcv     ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:41201_c_dcv
   [junit4]   2> 1932291 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [n:127.0.0.1:41201_c_dcv     ] o.a.s.c.Overseer Overseer (id=72136028182478852-127.0.0.1:41201_c_dcv-n_0000000000) starting
   [junit4]   2> 1932297 INFO  (OverseerStateUpdate-72136028182478852-127.0.0.1:41201_c_dcv-n_0000000000) [n:127.0.0.1:41201_c_dcv     ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:41201_c_dcv
   [junit4]   2> 1932297 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [n:127.0.0.1:41201_c_dcv     ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41201_c_dcv
   [junit4]   2> 1932307 INFO  (zkCallback-9463-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1932309 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [n:127.0.0.1:41201_c_dcv     ] o.a.s.p.PackageLoader /packages.json updated to version -1
   [junit4]   2> 1932309 WARN  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [n:127.0.0.1:41201_c_dcv     ] o.a.s.c.CoreContainer Not all security plugins configured!  authentication=disabled authorization=disabled.  Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external.  See https://s.apache.org/solrsecurity for more info
   [junit4]   2> 1932323 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [n:127.0.0.1:41201_c_dcv     ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 1932342 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [n:127.0.0.1:41201_c_dcv     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@67d3da70
   [junit4]   2> 1932350 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [n:127.0.0.1:41201_c_dcv     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@67d3da70
   [junit4]   2> 1932350 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [n:127.0.0.1:41201_c_dcv     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@67d3da70
   [junit4]   2> 1932351 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [n:127.0.0.1:41201_c_dcv     ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_BCECCDC0F80649F5-001/control-001/cores
   [junit4]   2> 1932366 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1932366 INFO  (zkConnectionManagerCallback-9473-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1932367 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1932368 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1932369 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45638/solr ready
   [junit4]   2> 1932370 INFO  (qtp918501496-26599) [n:127.0.0.1:41201_c_dcv     ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:41201_c_dcv&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1932377 INFO  (OverseerThreadFactory-4358-thread-1-processing-n:127.0.0.1:41201_c_dcv) [n:127.0.0.1:41201_c_dcv     ] o.a.s.c.a.c.CreateCollectionCmd Create collection control_collection
   [junit4]   2> 1932498 INFO  (qtp918501496-26598) [n:127.0.0.1:41201_c_dcv    x:control_collection_shard1_replica_n1 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1932499 INFO  (qtp918501496-26598) [n:127.0.0.1:41201_c_dcv    x:control_collection_shard1_replica_n1 ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 1933510 INFO  (qtp918501496-26598) [n:127.0.0.1:41201_c_dcv c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.0.0
   [junit4]   2> 1933524 INFO  (qtp918501496-26598) [n:127.0.0.1:41201_c_dcv c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
   [junit4]   2> 1933647 INFO  (qtp918501496-26598) [n:127.0.0.1:41201_c_dcv c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 1933663 INFO  (qtp918501496-26598) [n:127.0.0.1:41201_c_dcv c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
   [junit4]   2> 1933663 INFO  (qtp918501496-26598) [n:127.0.0.1:41201_c_dcv c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@67d3da70
   [junit4]   2> 1933663 INFO  (qtp918501496-26598) [n:127.0.0.1:41201_c_dcv c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_BCECCDC0F80649F5-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_BCECCDC0F80649F5-001/control-001/cores/control_collection_shard1_replica_n1/data/]
   [junit4]   2> 1933667 INFO  (qtp918501496-26598) [n:127.0.0.1:41201_c_dcv c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@799c42d2
   [junit4]   2> 1933676 WARN  (qtp918501496-26598) [n:127.0.0.1:41201_c_dcv c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A, b=B}}}
   [junit4]   2> 1933725 INFO  (qtp918501496-26598) [n:127.0.0.1:41201_c_dcv c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1933725 INFO  (qtp918501496-26598) [n:127.0.0.1:41201_c_dcv c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1933727 INFO  (qtp918501496-26598) [n:127.0.0.1:41201_c_dcv c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms; 
   [junit4]   2> 1933727 INFO  (qtp918501496-26598) [n:127.0.0.1:41201_c_dcv c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1933728 INFO  (qtp918501496-26598) [n:127.0.0.1:41201_c_dcv c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=29, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 1933728 INFO  (qtp918501496-26598) [n:127.0.0.1:41201_c_dcv c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.s.SolrIndexSearcher Opening [Searcher@4f9ae3ff[control_collection_shard1_replica_n1] main]
   [junit4]   2> 1933730 INFO  (qtp918501496-26598) [n:127.0.0.1:41201_c_dcv c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1933731 INFO  (qtp918501496-26598) [n:127.0.0.1:41201_c_dcv c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1933731 INFO  (qtp918501496-26598) [n:127.0.0.1:41201_c_dcv c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 1933731 INFO  (qtp918501496-26598) [n:127.0.0.1:41201_c_dcv c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1656143244807897088
   [junit4]   2> 1933734 INFO  (searcherExecutor-4363-thread-1-processing-n:127.0.0.1:41201_c_dcv x:control_collection_shard1_replica_n1 c:control_collection s:shard1) [n:127.0.0.1:41201_c_dcv c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@4f9ae3ff[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1933737 INFO  (qtp918501496-26598) [n:127.0.0.1:41201_c_dcv c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.ZkShardTerms Successful update of terms at /collections/control_collection/terms/shard1 to Terms{values={core_node2=0}, version=0}
   [junit4]   2> 1933737 INFO  (qtp918501496-26598) [n:127.0.0.1:41201_c_dcv c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/control_collection/leaders/shard1
   [junit4]   2> 1933740 INFO  (qtp918501496-26598) [n:127.0.0.1:41201_c_dcv c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1933740 INFO  (qtp918501496-26598) [n:127.0.0.1:41201_c_dcv c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1933740 INFO  (qtp918501496-26598) [n:127.0.0.1:41201_c_dcv c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:41201/c_dcv/control_collection_shard1_replica_n1/
   [junit4]   2> 1933740 INFO  (qtp918501496-26598) [n:127.0.0.1:41201_c_dcv c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 1933740 INFO  (qtp918501496-26598) [n:127.0.0.1:41201_c_dcv c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.SyncStrategy http://127.0.0.1:41201/c_dcv/control_collection_shard1_replica_n1/ has no replicas
   [junit4]   2> 1933740 INFO  (qtp918501496-26598) [n:127.0.0.1:41201_c_dcv c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/control_collection/leaders/shard1/leader after winning as /collections/control_collection/leader_elect/shard1/election/72136028182478852-core_node2-n_0000000000
   [junit4]   2> 1933742 INFO  (qtp918501496-26598) [n:127.0.0.1:41201_c_dcv c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:41201/c_dcv/control_collection_shard1_replica_n1/ shard1
   [junit4]   2> 1933844 INFO  (zkCallback-9463-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1933844 INFO  (zkCallback-9463-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1933844 INFO  (qtp918501496-26598) [n:127.0.0.1:41201_c_dcv c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1933846 INFO  (qtp918501496-26598) [n:127.0.0.1:41201_c_dcv c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1348
   [junit4]   2> 1933847 INFO  (qtp918501496-26599) [n:127.0.0.1:41201_c_dcv     ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
   [junit4]   2> 1933946 INFO  (zkCallback-9463-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1933946 INFO  (zkCallback-9463-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1933946 INFO  (zkCallback-9463-thread-3) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1933947 INFO  (qtp918501496-26599) [n:127.0.0.1:41201_c_dcv     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:41201_c_dcv&wt=javabin&version=2} status=0 QTime=1576
   [junit4]   2> 1933947 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.AbstractFullDistribZkTestBase Waiting to see 1 active replicas in collection: control_collection
   [junit4]   2> 1934055 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1934056 INFO  (zkConnectionManagerCallback-9479-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1934056 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1934057 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1934058 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45638/solr ready
   [junit4]   2> 1934058 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1934059 INFO  (qtp918501496-26600) [n:127.0.0.1:41201_c_dcv     ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=2&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1934061 INFO  (OverseerThreadFactory-4358-thread-2-processing-n:127.0.0.1:41201_c_dcv) [n:127.0.0.1:41201_c_dcv     ] o.a.s.c.a.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 1934061 INFO  (OverseerCollectionConfigSetProcessor-72136028182478852-127.0.0.1:41201_c_dcv-n_0000000000) [n:127.0.0.1:41201_c_dcv     ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1934265 WARN  (OverseerThreadFactory-4358-thread-2-processing-n:127.0.0.1:41201_c_dcv) [n:127.0.0.1:41201_c_dcv     ] o.a.s.c.a.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
   [junit4]   2> 1934266 INFO  (qtp918501496-26600) [n:127.0.0.1:41201_c_dcv     ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
   [junit4]   2> 1934267 INFO  (qtp918501496-26600) [n:127.0.0.1:41201_c_dcv     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=2&wt=javabin&version=2} status=0 QTime=208
   [junit4]   2> 1934267 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.SolrCloudTestCase active slice count: 1 expected:1
   [junit4]   2> 1934267 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.SolrCloudTestCase active replica count: 0 expected replica count: 0
   [junit4]   2> 1934267 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.SolrCloudTestCase active slice count: 1 expected:1
   [junit4]   2> 1934268 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.SolrCloudTestCase active replica count: 0 expected replica count: 0
   [junit4]   2> 1934268 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.SolrCloudTestCase active slice count: 1 expected:1
   [junit4]   2> 1934268 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.SolrCloudTestCase active replica count: 0 expected replica count: 0
   [junit4]   2> 1934268 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.AbstractFullDistribZkTestBase Creating jetty instances pullReplicaCount=0 numOtherReplicas=2
   [junit4]   2> 1934638 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_BCECCDC0F80649F5-001/shard-1-001 of type NRT
   [junit4]   2> 1934638 WARN  (closeThreadPool-9480-thread-1) [     ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
   [junit4]   2> 1934639 INFO  (closeThreadPool-9480-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 1934639 INFO  (closeThreadPool-9480-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 1934639 INFO  (closeThreadPool-9480-thread-1) [     ] o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z; git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 11.0.4+10-LTS
   [junit4]   2> 1934639 INFO  (closeThreadPool-9480-thread-1) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1934639 INFO  (closeThreadPool-9480-thread-1) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1934640 INFO  (closeThreadPool-9480-thread-1) [     ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 1934640 INFO  (closeThreadPool-9480-thread-1) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@292d7769{/c_dcv,null,AVAILABLE}
   [junit4]   2> 1934640 INFO  (closeThreadPool-9480-thread-1) [     ] o.e.j.s.AbstractConnector Started ServerConnector@dd031d0{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:35095}
   [junit4]   2> 1934640 INFO  (closeThreadPool-9480-thread-1) [     ] o.e.j.s.Server Started @1934681ms
   [junit4]   2> 1934640 INFO  (closeThreadPool-9480-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/c_dcv, solrconfig=solrconfig.xml, solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_BCECCDC0F80649F5-001/tempDir-001/jetty1, hostPort=35095, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_BCECCDC0F80649F5-001/shard-1-001/cores}
   [junit4]   2> 1934640 ERROR (closeThreadPool-9480-thread-1) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1934640 INFO  (closeThreadPool-9480-thread-1) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 1934641 INFO  (closeThreadPool-9480-thread-1) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 9.0.0
   [junit4]   2> 1934641 INFO  (closeThreadPool-9480-thread-1) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1934641 INFO  (closeThreadPool-9480-thread-1) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1934641 INFO  (closeThreadPool-9480-thread-1) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2020-01-19T08:07:57.773083Z
   [junit4]   2> 1934641 INFO  (closeThreadPool-9480-thread-1) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1934643 INFO  (zkConnectionManagerCallback-9482-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1934643 INFO  (closeThreadPool-9480-thread-1) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1934756 INFO  (closeThreadPool-9480-thread-1) [     ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 1934756 INFO  (closeThreadPool-9480-thread-1) [     ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_BCECCDC0F80649F5-001/shard-1-001/solr.xml
   [junit4]   2> 1934758 INFO  (closeThreadPool-9480-thread-1) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 1934758 INFO  (closeThreadPool-9480-thread-1) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 1934760 INFO  (closeThreadPool-9480-thread-1) [     ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@67d3da70, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 1935005 INFO  (closeThreadPool-9480-thread-1) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
   [junit4]   2> 1935006 WARN  (closeThreadPool-9480-thread-1) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@5c57bb56[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1935006 WARN  (closeThreadPool-9480-thread-1) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@5c57bb56[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1935007 WARN  (closeThreadPool-9480-thread-1) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@1045592e[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1935008 WARN  (closeThreadPool-9480-thread-1) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@1045592e[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1935017 INFO  (closeThreadPool-9480-thread-1) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45638/solr
   [junit4]   2> 1935018 INFO  (closeThreadPool-9480-thread-1) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1935018 INFO  (zkConnectionManagerCallback-9489-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1935018 INFO  (closeThreadPool-9480-thread-1) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1935062 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_BCECCDC0F80649F5-001/shard-2-001 of type NRT
   [junit4]   2> 1935063 WARN  (closeThreadPool-9480-thread-2) [     ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
   [junit4]   2> 1935063 INFO  (closeThreadPool-9480-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 1935063 INFO  (closeThreadPool-9480-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 1935063 INFO  (closeThreadPool-9480-thread-2) [     ] o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z; git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 11.0.4+10-LTS
   [junit4]   2> 1935064 INFO  (closeThreadPool-9480-thread-2) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1935064 INFO  (closeThreadPool-9480-thread-2) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1935064 INFO  (closeThreadPool-9480-thread-2) [     ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 1935064 INFO  (closeThreadPool-9480-thread-2) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1e20ab22{/c_dcv,null,AVAILABLE}
   [junit4]   2> 1935064 INFO  (closeThreadPool-9480-thread-2) [     ] o.e.j.s.AbstractConnector Started ServerConnector@7ce2a145{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:43404}
   [junit4]   2> 1935064 INFO  (closeThreadPool-9480-thread-2) [     ] o.e.j.s.Server Started @1935105ms
   [junit4]   2> 1935064 INFO  (closeThreadPool-9480-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/c_dcv, solrconfig=solrconfig.xml, solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_BCECCDC0F80649F5-001/tempDir-001/jetty2, hostPort=43404, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_BCECCDC0F80649F5-001/shard-2-001/cores}
   [junit4]   2> 1935065 ERROR (closeThreadPool-9480-thread-2) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1935065 INFO  (closeThreadPool-9480-thread-2) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 1935065 INFO  (closeThreadPool-9480-thread-2) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 9.0.0
   [junit4]   2> 1935065 INFO  (closeThreadPool-9480-thread-2) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1935065 INFO  (closeThreadPool-9480-thread-2) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1935065 INFO  (closeThreadPool-9480-thread-2) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2020-01-19T08:07:58.197200Z
   [junit4]   2> 1935066 INFO  (closeThreadPool-9480-thread-2) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1935066 INFO  (zkConnectionManagerCallback-9491-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1935066 INFO  (closeThreadPool-9480-thread-2) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1935120 INFO  (closeThreadPool-9480-thread-1) [n:127.0.0.1:35095_c_dcv     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1935121 INFO  (zkConnectionManagerCallback-9493-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1935121 INFO  (closeThreadPool-9480-thread-1) [n:127.0.0.1:35095_c_dcv     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1935125 INFO  (closeThreadPool-9480-thread-1) [n:127.0.0.1:35095_c_dcv     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1935138 INFO  (closeThreadPool-9480-thread-1) [n:127.0.0.1:35095_c_dcv     ] o.a.s.c.ZkController Publish node=127.0.0.1:35095_c_dcv as DOWN
   [junit4]   2> 1935138 INFO  (closeThreadPool-9480-thread-1) [n:127.0.0.1:35095_c_dcv     ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 1935138 INFO  (closeThreadPool-9480-thread-1) [n:127.0.0.1:35095_c_dcv     ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35095_c_dcv
   [junit4]   2> 1935139 INFO  (zkCallback-9463-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1935140 INFO  (zkCallback-9478-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1935140 INFO  (zkCallback-9492-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1935142 INFO  (closeThreadPool-9480-thread-1) [n:127.0.0.1:35095_c_dcv     ] o.a.s.p.PackageLoader /packages.json updated to version -1
   [junit4]   2> 1935142 WARN  (closeThreadPool-9480-thread-1) [n:127.0.0.1:35095_c_dcv     ] o.a.s.c.CoreContainer Not all security plugins configured!  authentication=disabled authorization=disabled.  Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external.  See https://s.apache.org/solrsecurity for more info
   [junit4]   2> 1935165 INFO  (closeThreadPool-9480-thread-1) [n:127.0.0.1:35095_c_dcv     ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 1935184 INFO  (closeThreadPool-9480-thread-1) [n:127.0.0.1:35095_c_dcv     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@67d3da70
   [junit4]   2> 1935193 INFO  (closeThreadPool-9480-thread-1) [n:127.0.0.1:35095_c_dcv     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@67d3da70
   [junit4]   2> 1935193 INFO  (closeThreadPool-9480-thread-1) [n:127.0.0.1:35095_c_dcv     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@67d3da70
   [junit4]   2> 1935195 INFO  (closeThreadPool-9480-thread-1) [n:127.0.0.1:35095_c_dcv     ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_BCECCDC0F80649F5-001/shard-1-001/cores
   [junit4]   2> 1935288 INFO  (closeThreadPool-9480-thread-2) [     ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 1935288 INFO  (closeThreadPool-9480-thread-2) [     ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_BCECCDC0F80649F5-001/shard-2-001/solr.xml
   [junit4]   2> 1935290 INFO  (closeThreadPool-9480-thread-2) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 1935290 INFO  (closeThreadPool-9480-thread-2) [     ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 1935292 INFO  (closeThreadPool-9480-thread-2) [     ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@67d3da70, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 1935317 INFO  (closeThreadPool-9480-thread-1) [     ] o.a.s.c.AbstractFullDistribZkTestBase waitForLiveNode: 127.0.0.1:35095_c_dcv
   [junit4]   2> 1935398 INFO  (closeThreadPool-9480-thread-2) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
   [junit4]   2> 1935399 WARN  (closeThreadPool-9480-thread-2) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@7f7697c1[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1935399 WARN  (closeThreadPool-9480-thread-2) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@7f7697c1[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1935401 WARN  (closeThreadPool-9480-thread-2) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@717d8ced[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1935401 WARN  (closeThreadPool-9480-thread-2) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@717d8ced[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1935402 INFO  (closeThreadPool-9480-thread-2) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45638/solr
   [junit4]   2> 1935402 INFO  (closeThreadPool-9480-thread-2) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1935403 INFO  (zkConnectionManagerCallback-9504-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1935403 INFO  (closeThreadPool-9480-thread-2) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1935505 INFO  (closeThreadPool-9480-thread-2) [n:127.0.0.1:43404_c_dcv     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1935506 INFO  (zkConnectionManagerCallback-9506-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1935506 INFO  (closeThreadPool-9480-thread-2) [n:127.0.0.1:43404_c_dcv     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1935510 INFO  (closeThreadPool-9480-thread-2) [n:127.0.0.1:43404_c_dcv     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1935513 INFO  (closeThreadPool-9480-thread-2) [n:127.0.0.1:43404_c_dcv     ] o.a.s.c.ZkController Publish node=127.0.0.1:43404_c_dcv as DOWN
   [junit4]   2> 1935514 INFO  (closeThreadPool-9480-thread-2) [n:127.0.0.1:43404_c_dcv     ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 1935514 INFO  (closeThreadPool-9480-thread-2) [n:127.0.0.1:43404_c_dcv     ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43404_c_dcv
   [junit4]   2> 1935515 INFO  (zkCallback-9463-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1935515 INFO  (zkCallback-9478-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1935515 INFO  (zkCallback-9492-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1935515 INFO  (zkCallback-9505-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1935517 INFO  (closeThreadPool-9480-thread-2) [n:127.0.0.1:43404_c_dcv     ] o.a.s.p.PackageLoader /packages.json updated to version -1
   [junit4]   2> 1935517 WARN  (closeThreadPool-9480-thread-2) [n:127.0.0.1:43404_c_dcv     ] o.a.s.c.CoreContainer Not all security plugins configured!  authentication=disabled authorization=disabled.  Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external.  See https://s.apache.org/solrsecurity for more info
   [junit4]   2> 1935532 INFO  (closeThreadPool-9480-thread-2) [n:127.0.0.1:43404_c_dcv     ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 1935551 INFO  (closeThreadPool-9480-thread-2) [n:127.0.0.1:43404_c_dcv     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@67d3da70
   [junit4]   2> 1935560 INFO  (closeThreadPool-9480-thread-2) [n:127.0.0.1:43404_c_dcv     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@67d3da70
   [junit4]   2> 1935560 INFO  (closeThreadPool-9480-thread-2) [n:127.0.0.1:43404_c_dcv     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@67d3da70
   [junit4]   2> 1935561 INFO  (closeThreadPool-9480-thread-2) [n:127.0.0.1:43404_c_dcv     ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_BCECCDC0F80649F5-001/shard-2-001/cores
   [junit4]   2> 1935589 INFO  (closeThreadPool-9480-thread-2) [     ] o.a.s.c.AbstractFullDistribZkTestBase waitForLiveNode: 127.0.0.1:43404_c_dcv
   [junit4]   2> 1935591 INFO  (qtp166704181-26656) [n:127.0.0.1:35095_c_dcv     ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:35095_c_dcv&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1935591 INFO  (qtp166704181-26655) [n:127.0.0.1:35095_c_dcv     ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:43404_c_dcv&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1935602 INFO  (OverseerCollectionConfigSetProcessor-72136028182478852-127.0.0.1:41201_c_dcv-n_0000000000) [n:127.0.0.1:41201_c_dcv     ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1935603 INFO  (OverseerThreadFactory-4358-thread-3-processing-n:127.0.0.1:41201_c_dcv) [n:127.0.0.1:41201_c_dcv c:collection1 s:shard1   ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:35095_c_dcv for creating new replica of shard shard1 for collection collection1
   [junit4]   2> 1935616 INFO  (OverseerThreadFactory-4358-thread-3-processing-n:127.0.0.1:41201_c_dcv) [n:127.0.0.1:41201_c_dcv c:collection1 s:shard1   ] o.a.s.c.a.c.AddReplicaCmd Returning CreateReplica command.
   [junit4]   2> 1935619 INFO  (qtp166704181-26657) [n:127.0.0.1:35095_c_dcv    x:collection1_shard1_replica_n1 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n1&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1936634 INFO  (qtp166704181-26657) [n:127.0.0.1:35095_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n1 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.0.0
   [junit4]   2> 1936656 INFO  (qtp166704181-26657) [n:127.0.0.1:35095_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n1 ] o.a.s.s.IndexSchema [collection1_shard1_replica_n1] Schema name=test
   [junit4]   2> 1936722 INFO  (qtp166704181-26657) [n:127.0.0.1:35095_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n1 ] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 1936737 INFO  (qtp166704181-26657) [n:127.0.0.1:35095_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n1 ] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n1' using configuration from collection collection1, trusted=true
   [junit4]   2> 1936737 INFO  (qtp166704181-26657) [n:127.0.0.1:35095_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n1 ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_n1' (registry 'solr.core.collection1.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@67d3da70
   [junit4]   2> 1936737 INFO  (qtp166704181-26657) [n:127.0.0.1:35095_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n1 ] o.a.s.c.SolrCore [[collection1_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_BCECCDC0F80649F5-001/shard-1-001/cores/collection1_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_BCECCDC0F80649F5-001/shard-1-001/cores/collection1_shard1_replica_n1/data/]
   [junit4]   2> 1936740 INFO  (qtp166704181-26657) [n:127.0.0.1:35095_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n1 ] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@72f20cab
   [junit4]   2> 1936750 WARN  (qtp166704181-26657) [n:127.0.0.1:35095_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n1 ] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A, b=B}}}
   [junit4]   2> 1936810 INFO  (qtp166704181-26657) [n:127.0.0.1:35095_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n1 ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1936810 INFO  (qtp166704181-26657) [n:127.0.0.1:35095_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n1 ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1936811 INFO  (qtp166704181-26657) [n:127.0.0.1:35095_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n1 ] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms; 
   [junit4]   2> 1936811 INFO  (qtp166704181-26657) [n:127.0.0.1:35095_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n1 ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1936812 INFO  (qtp166704181-26657) [n:127.0.0.1:35095_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n1 ] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=29, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 1936813 INFO  (qtp166704181-26657) [n:127.0.0.1:35095_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n1 ] o.a.s.s.SolrIndexSearcher Opening [Searcher@13ba02c3[collection1_shard1_replica_n1] main]
   [junit4]   2> 1936814 INFO  (qtp166704181-26657) [n:127.0.0.1:35095_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n1 ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1936815 INFO  (qtp166704181-26657) [n:127.0.0.1:35095_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n1 ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1936815 INFO  (qtp166704181-26657) [n:127.0.0.1:35095_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n1 ] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 1936815 INFO  (qtp166704181-26657) [n:127.0.0.1:35095_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n1 ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1656143248041705472
   [junit4]   2> 1936818 INFO  (searcherExecutor-4386-thread-1-processing-n:127.0.0.1:35095_c_dcv x:collection1_shard1_replica_n1 c:collection1 s:shard1) [n:127.0.0.1:35095_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n1 ] o.a.s.c.SolrCore [collection1_shard1_replica_n1] Registered new searcher Searcher@13ba02c3[collection1_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1936821 INFO  (qtp166704181-26657) [n:127.0.0.1:35095_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n1 ] o.a.s.c.ZkShardTerms Successful update of terms at /collections/collection1/terms/shard1 to Terms{values={core_node2=0}, version=0}
   [junit4]   2> 1936821 INFO  (qtp166704181-26657) [n:127.0.0.1:35095_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/collection1/leaders/shard1
   [junit4]   2> 1936824 INFO  (qtp166704181-26657) [n:127.0.0.1:35095_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1936824 INFO  (qtp166704181-26657) [n:127.0.0.1:35095_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1936824 INFO  (qtp166704181-26657) [n:127.0.0.1:35095_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n1 ] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:35095/c_dcv/collection1_shard1_replica_n1/
   [junit4]   2> 1936824 INFO  (qtp166704181-26657) [n:127.0.0.1:35095_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n1 ] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 1936824 INFO  (qtp166704181-26657) [n:127.0.0.1:35095_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n1 ] o.a.s.c.SyncStrategy http://127.0.0.1:35095/c_dcv/collection1_shard1_replica_n1/ has no replicas
   [junit4]   2> 1936824 INFO  (qtp166704181-26657) [n:127.0.0.1:35095_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/collection1/leaders/shard1/leader after winning as /collections/collection1/leader_elect/shard1/election/72136028182478858-core_node2-n_0000000000
   [junit4]   2> 1936826 INFO  (qtp166704181-26657) [n:127.0.0.1:35095_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:35095/c_dcv/collection1_shard1_replica_n1/ shard1
   [junit4]   2> 1936928 INFO  (zkCallback-9492-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1936928 INFO  (zkCallback-9492-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1936928 INFO  (qtp166704181-26657) [n:127.0.0.1:35095_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n1 ] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1936930 INFO  (qtp166704181-26657) [n:127.0.0.1:35095_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n1 ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n1&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1311
   [junit4]   2> 1936932 INFO  (qtp166704181-26656) [n:127.0.0.1:35095_c_dcv c:collection1    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:35095_c_dcv&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2} status=0 QTime=1341
   [junit4]   2> 1937030 INFO  (zkCallback-9492-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1937030 INFO  (zkCallback-9492-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1937030 INFO  (zkCallback-9492-thread-3) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1937604 INFO  (OverseerCollectionConfigSetProcessor-72136028182478852-127.0.0.1:41201_c_dcv-n_0000000000) [n:127.0.0.1:41201_c_dcv     ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1937605 INFO  (OverseerThreadFactory-4358-thread-4-processing-n:127.0.0.1:41201_c_dcv) [n:127.0.0.1:41201_c_dcv c:collection1 s:shard1   ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:43404_c_dcv for creating new replica of shard shard1 for collection collection1
   [junit4]   2> 1937606 INFO  (OverseerThreadFactory-4358-thread-4-processing-n:127.0.0.1:41201_c_dcv) [n:127.0.0.1:41201_c_dcv c:collection1 s:shard1   ] o.a.s.c.a.c.AddReplicaCmd Returning CreateReplica command.
   [junit4]   2> 1937609 INFO  (qtp2027764358-26677) [n:127.0.0.1:43404_c_dcv    x:collection1_shard1_replica_n3 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n3&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1937714 INFO  (zkCallback-9492-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1937714 INFO  (zkCallback-9492-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1937714 INFO  (zkCallback-9492-thread-3) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1938707 INFO  (qtp2027764358-26677) [n:127.0.0.1:43404_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n3 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.0.0
   [junit4]   2> 1938734 INFO  (qtp2027764358-26677) [n:127.0.0.1:43404_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n3 ] o.a.s.s.IndexSchema [collection1_shard1_replica_n3] Schema name=test
   [junit4]   2> 1938801 INFO  (qtp2027764358-26677) [n:127.0.0.1:43404_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n3 ] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 1938816 INFO  (qtp2027764358-26677) [n:127.0.0.1:43404_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n3 ] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n3' using configuration from collection collection1, trusted=true
   [junit4]   2> 1938816 INFO  (qtp2027764358-26677) [n:127.0.0.1:43404_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n3 ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_n3' (registry 'solr.core.collection1.shard1.replica_n3') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@67d3da70
   [junit4]   2> 1938816 INFO  (qtp2027764358-26677) [n:127.0.0.1:43404_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n3 ] o.a.s.c.SolrCore [[collection1_shard1_replica_n3] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_BCECCDC0F80649F5-001/shard-2-001/cores/collection1_shard1_replica_n3], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_BCECCDC0F80649F5-001/shard-2-001/cores/collection1_shard1_replica_n3/data/]
   [junit4]   2> 1938819 INFO  (qtp2027764358-26677) [n:127.0.0.1:43404_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n3 ] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@42f08e26
   [junit4]   2> 1938828 WARN  (qtp2027764358-26677) [n:127.0.0.1:43404_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n3 ] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A, b=B}}}
   [junit4]   2> 1938877 INFO  (qtp2027764358-26677) [n:127.0.0.1:43404_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n3 ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1938877 INFO  (qtp2027764358-26677) [n:127.0.0.1:43404_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n3 ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1938878 INFO  (qtp2027764358-26677) [n:127.0.0.1:43404_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n3 ] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms; 
   [junit4]   2> 1938878 INFO  (qtp2027764358-26677) [n:127.0.0.1:43404_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n3 ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1938879 INFO  (qtp2027764358-26677) [n:127.0.0.1:43404_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n3 ] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=29, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 1938880 INFO  (qtp2027764358-26677) [n:127.0.0.1:43404_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n3 ] o.a.s.s.SolrIndexSearcher Opening [Searcher@6537e32a[collection1_shard1_replica_n3] main]
   [junit4]   2> 1938881 INFO  (qtp2027764358-26677) [n:127.0.0.1:43404_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n3 ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1938882 INFO  (qtp2027764358-26677) [n:127.0.0.1:43404_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n3 ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1938882 INFO  (qtp2027764358-26677) [n:127.0.0.1:43404_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n3 ] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 1938882 INFO  (qtp2027764358-26677) [n:127.0.0.1:43404_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n3 ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1656143250209112064
   [junit4]   2> 1938885 INFO  (searcherExecutor-4391-thread-1-processing-n:127.0.0.1:43404_c_dcv x:collection1_shard1_replica_n3 c:collection1 s:shard1) [n:127.0.0.1:43404_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n3 ] o.a.s.c.SolrCore [collection1_shard1_replica_n3] Registered new searcher Searcher@6537e32a[collection1_shard1_replica_n3] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1938888 INFO  (qtp2027764358-26677) [n:127.0.0.1:43404_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n3 ] o.a.s.c.ZkShardTerms Successful update of terms at /collections/collection1/terms/shard1 to Terms{values={core_node2=0, core_node4=0}, version=1}
   [junit4]   2> 1938900 INFO  (qtp2027764358-26677) [n:127.0.0.1:43404_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n3 ] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/collection1/leaders/shard1
   [junit4]   2> 1938911 INFO  (qtp2027764358-26677) [n:127.0.0.1:43404_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n3 ] o.a.s.c.ZkController Core needs to recover:collection1_shard1_replica_n3
   [junit4]   2> 1938911 INFO  (updateExecutor-9500-thread-1-processing-n:127.0.0.1:43404_c_dcv x:collection1_shard1_replica_n3 c:collection1 s:shard1) [n:127.0.0.1:43404_c_dcv c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n3 ] o.a.s.u.DefaultSolrCoreState Running recovery
   [junit4]   2> 1938912 INFO  (recoveryExecutor-9502-thread-1-processing-n:127.0.0.1:43404_c_dcv x:collection1_shard1_replica_n3 c:collection1 s:shard1 r:core_node4) [n:127.0.0.1:43404_c_dcv c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n3 ] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
   [junit4]   2> 1938912 INFO  (recoveryExecutor-9502-thread-1-processing-n:127.0.0.1:43404_c_dcv x:collection1_shard1_replica_n3 c:collection1 s:shard1 r:core_node4) [n:127.0.0.1:43404_c_dcv c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n3 ] o.a.s.c.RecoveryStrategy startupVersions is empty
   [junit4]   2> 1938913 INFO  (qtp2027764358-26677) [n:127.0.0.1:43404_c_dcv c:collection1 s:shard1  x:collection1_shard1_replica_n3 ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n3&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1303
   [junit4]   2> 1938915 INFO  (qtp166704181-26655) [n:127.0.0.1:35095_c_dcv c:collection1    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:43404_c_dcv&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2} status=0 QTime=3323
   [junit4]   2> 1938915 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.AbstractFullDistribZkTestBase Waiting to see 2 active replicas in collection: collection1
   [junit4]   2> 1938930 INFO  (qtp166704181-26656) [n:127.0.0.1:35095_c_dcv c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica_n1 ] o.a.s.c.S.Request [collection1_shard1_replica_n1]  webapp=/c_dcv path=/admin/ping params={wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 1938930 INFO  (qtp166704181-26656) [n:127.0.0.1:35095_c_dcv c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica_n1 ] o.a.s.c.S.Request [collection1_shard1_replica_n1]  webapp=/c_dcv path=/admin/ping params={wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1938930 INFO  (recoveryExecutor-9502-thread-1-processing-n:127.0.0.1:43404_c_dcv x:collection1_shard1_replica_n3 c:collection1 s:shard1 r:core_node4) [n:127.0.0.1:43404_c_dcv c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n3 ] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1_shard1_replica_n3]
   [junit4]   2> 1938930 INFO  (recoveryExecutor-9502-thread-1-processing-n:127.0.0.1:43404_c_dcv x:collection1_shard1_replica_n3 c:collection1 s:shard1 r:core_node4) [n:127.0.0.1:43404_c_dcv c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n3 ] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
   [junit4]   2> 1938930 INFO  (recoveryExecutor-9502-thread-1-processing-n:127.0.0.1:43404_c_dcv x:collection1_shard1_replica_n3 c:collection1 s:shard1 r:core_node4) [n:127.0.0.1:43404_c_dcv c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n3 ] o.a.s.c.RecoveryStrategy Publishing state of core [collection1_shard1_replica_n3] as recovering, leader is [http://127.0.0.1:35095/c_dcv/collection1_shard1_replica_n1/] and I am [http://127.0.0.1:43404/c_dcv/collection1_shard1_replica_n3/]
   [junit4]   2> 1938937 INFO  (recoveryExecutor-9502-thread-1-processing-n:127.0.0.1:43404_c_dcv x:collection1_shard1_replica_n3 c:collection1 s:shard1 r:core_node4) [n:127.0.0.1:43404_c_dcv c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n3 ] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:35095/c_dcv]; [WaitForState: action=PREPRECOVERY&core=collection1_shard1_replica_n1&nodeName=127.0.0.1:43404_c_dcv&coreNodeName=core_node4&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
   [junit4]   2> 1938937 INFO  (qtp166704181-26655) [n:127.0.0.1:35095_c_dcv    x:collection1_shard1_replica_n1 ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node4, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
   [junit4]   2> 1938937 INFO  (qtp166704181-26655) [n:127.0.0.1:35095_c_dcv    x:collection1_shard1_replica_n1 ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1_shard1_replica_n1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:43404_c_dcv, coreNodeName=core_node4, onlyIfActiveCheckResult=false, nodeProps: core_node4:{
   [junit4]   2>   "core":"collection1_shard1_replica_n3",
   [junit4]   2>   "base_url":"http://127.0.0.1:43404/c_dcv",
   [junit4]   2>   "node_name":"127.0.0.1:43404_c_dcv",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "type":"NRT"}
   [junit4]   2> 1938937 INFO  (qtp166704181-26655) [n:127.0.0.1:35095_c_dcv    x:collection1_shard1_replica_n1 ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1_shard1_replica_n1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:43404_c_dcv, coreNodeName=core_node4, onlyIfActiveCheckResult=false, nodeProps: core_node4:{
   [junit4]   2>   "core":"collection1_shard1_replica_n3",
   [junit4]   2>   "base_url":"http://127.0.0.1:43404/c_dcv",
   [junit4]   2>   "node_name":"127.0.0.1:43404_c_dcv",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "type":"NRT"}
   [junit4]   2> 1938937 INFO  (qtp166704181-26655) [n:127.0.0.1:35095_c_dcv    x:collection1_shard1_replica_n1 ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1_shard1_replica_n1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:43404_c_dcv, coreNodeName=core_node4, onlyIfActiveCheckResult=false, nodeProps: core_node4:{
   [junit4]   2>   "core":"collection1_shard1_replica_n3",
   [junit4]   2>   "base_url":"http://127.0.0.1:43404/c_dcv",
   [junit4]   2>   "node_name":"127.0.0.1:43404_c_dcv",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "type":"NRT"}
   [junit4]   2> 1939032 INFO  (zkCallback-9505-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1939032 INFO  (zkCallback-9492-thread-3) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1939032 INFO  (zkCallback-9492-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1939032 INFO  (zkCallback-9478-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1939032 INFO  (zkCallback-9492-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1939032 INFO  (zkCallback-9505-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1939032 INFO  (zkCallback-9505-thread-3) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1939033 INFO  (watches-9494-thread-1) [     ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1_shard1_replica_n1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:43404_c_dcv, coreNodeName=core_node4, onlyIfActiveCheckResult=false, nodeProps: core_node4:{
   [junit4]   2>   "core":"collection1_shard1_replica_n3",
   [junit4]   2>   "base_url":"http://127.0.0.1:43404/c_dcv",
   [junit4]   2>   "node_name":"127.0.0.1:43404_c_dcv",
   [junit4]   2>   "state":"recovering",
   [junit4]   2>   "type":"NRT"}
   [junit4]   2> 1939033 INFO  (qtp166704181-26655) [n:127.0.0.1:35095_c_dcv    x:collection1_shard1_replica_n1 ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:43404_c_dcv&onlyIfLeaderActive=true&core=collection1_shard1_replica_n1&coreNodeName=core_node4&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=95
   [junit4]   2> 1939533 INFO  (recoveryExecutor-9502-thread-1-processing-n:127.0.0.1:43404_c_dcv x:collection1_shard1_replica_n3 c:collection1 s:shard1 r:core_node4) [n:127.0.0.1:43404_c_dcv c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n3 ] o.a.s.c.RecoveryStrategy Attempting to PeerSync from [http://127.0.0.1:35095/c_dcv/collection1_shard1_replica_n1/] - recoveringAfterStartup=[true]
   [junit4]   2> 1939533 WARN  (recoveryExecutor-9502-thread-1-processing-n:127.0.0.1:43404_c_dcv x:collection1_shard1_replica_n3 c:collection1 s:shard1 r:core_node4) [n:127.0.0.1:43404_c_dcv c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n3 ] o.a.s.u.PeerSyncWithLeader no frame of reference to tell if we've missed updates
   [junit4]   2> 1939533 INFO  (recoveryExecutor-9502-thread-1-processing-n:127.0.0.1:43404_c_dcv x:collection1_shard1_replica_n3 c:collection1 s:shard1 r:core_node4) [n:127.0.0.1:43404_c_dcv c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica

[...truncated too long message...]

-9530-thread-2) [     ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 1973037 INFO  (closeThreadPool-9530-thread-2) [     ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@777cd508: rootName = null, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@fe9e706
   [junit4]   2> 1973041 INFO  (closeThreadPool-9530-thread-2) [     ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 1973041 INFO  (closeThreadPool-9530-thread-2) [     ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@55f3d85d: rootName = null, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@393cfc66
   [junit4]   2> 1973042 INFO  (closeThreadPool-9530-thread-2) [     ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 1973042 INFO  (closeThreadPool-9530-thread-2) [     ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@67b8f1a6: rootName = null, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@74f3799b
   [junit4]   2> 1973042 INFO  (closeThreadPool-9530-thread-2) [     ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 1973057 INFO  (closeThreadPool-9537-thread-1) [     ] o.a.s.c.Overseer Overseer (id=72136028182478852-127.0.0.1:41201_c_dcv-n_0000000000) closing
   [junit4]   2> 1973057 INFO  (OverseerStateUpdate-72136028182478852-127.0.0.1:41201_c_dcv-n_0000000000) [n:127.0.0.1:41201_c_dcv     ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:41201_c_dcv
   [junit4]   2> 1973080 INFO  (OverseerAutoScalingTriggerThread-72136028182478852-127.0.0.1:41201_c_dcv-n_0000000000) [     ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 1973082 INFO  (closeThreadPool-9530-thread-5) [     ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 1973082 INFO  (closeThreadPool-9530-thread-5) [     ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@36889aba: rootName = null, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@393cfc66
   [junit4]   2> 1973084 INFO  (closeThreadPool-9530-thread-5) [     ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 1973084 INFO  (closeThreadPool-9530-thread-5) [     ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@2dc12570: rootName = null, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@74f3799b
   [junit4]   2> 1973084 INFO  (closeThreadPool-9530-thread-5) [     ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 1973169 INFO  (closeThreadPool-9537-thread-2) [     ] o.a.s.c.Overseer Overseer (id=72136028182478852-127.0.0.1:41201_c_dcv-n_0000000000) closing
   [junit4]   2> 1973185 INFO  (zkCallback-9505-thread-6) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
   [junit4]   2> 1973285 INFO  (closeThreadPool-9530-thread-2) [     ] o.a.s.c.Overseer Overseer (id=72136028182478852-127.0.0.1:41201_c_dcv-n_0000000000) closing
   [junit4]   2> 1973351 INFO  (closeThreadPool-9530-thread-2) [     ] o.e.j.s.AbstractConnector Stopped ServerConnector@82cfb3e{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:0}
   [junit4]   2> 1973376 INFO  (closeThreadPool-9530-thread-2) [     ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@791e8298{/c_dcv,null,UNAVAILABLE}
   [junit4]   2> 1973376 INFO  (closeThreadPool-9530-thread-2) [     ] o.e.j.s.session node0 Stopped scavenging
   [junit4]   2> 1973397 INFO  (closeThreadPool-9530-thread-5) [     ] o.e.j.s.AbstractConnector Stopped ServerConnector@7ce2a145{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:0}
   [junit4]   2> 1973397 INFO  (closeThreadPool-9530-thread-5) [     ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@1e20ab22{/c_dcv,null,UNAVAILABLE}
   [junit4]   2> 1973397 INFO  (closeThreadPool-9530-thread-5) [     ] o.e.j.s.session node0 Stopped scavenging
   [junit4]   2> 1973397 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.ZkTestServer Shutting down ZkTestServer.
   [junit4]   2> 1973602 WARN  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	10	/solr/collections/collection1/state.json
   [junit4]   2> 	3	/solr/collections/control_collection/state.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	5	/solr/overseer/queue
   [junit4]   2> 
   [junit4]   2> 1973603 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.ZkTestServer waitForServerDown: 127.0.0.1:45638
   [junit4]   2> 1973603 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:45638
   [junit4]   2> 1973603 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[BCECCDC0F80649F5]) [     ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 45638
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ChaosMonkeyNothingIsSafeTest -Dtests.method=test -Dtests.seed=BCECCDC0F80649F5 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=pl -Dtests.timezone=Antarctica/Palmer -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE 42.3s J1 | ChaosMonkeyNothingIsSafeTest.test <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: document count mismatch.  control=295 sum(shards)=0 cloudClient=0
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([BCECCDC0F80649F5:34B8F21A56FA240D]:0)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1522)
   [junit4]    > 	at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:266)
   [junit4]    > 	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]    > 	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]    > 	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]    > 	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1082)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1054)
   [junit4]    > 	at java.base/java.lang.Thread.run(Thread.java:834)
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_BCECCDC0F80649F5-001
   [junit4]   2> Jan 19, 2020 8:08:36 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene84): {rnd_b=PostingsFormat(name=LuceneVarGapDocFreqInterval), a_t=PostingsFormat(name=Direct), _root_=PostingsFormat(name=MockRandom), id=PostingsFormat(name=LuceneVarGapDocFreqInterval)}, docValues:{rnd_b=DocValuesFormat(name=Asserting), _version_=DocValuesFormat(name=Lucene80), a_t=DocValuesFormat(name=Lucene80), _root_=DocValuesFormat(name=Lucene80), a_i=DocValuesFormat(name=Asserting), id=DocValuesFormat(name=Asserting)}, maxPointsInLeafNode=1108, maxMBSortInHeap=5.456946702979798, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@32b62ef1), locale=pl, timezone=Antarctica/Palmer
   [junit4]   2> NOTE: Linux 4.4.0-170-generic amd64/Oracle Corporation 11.0.4 (64-bit)/cpus=8,threads=1,free=257354944,total=511705088
   [junit4]   2> NOTE: All tests run in this JVM: [BasicDistributedZk2Test, TestSuggestSpellingConverter, HdfsLockFactoryTest, TestFieldTypeCollectionResource, TestNumericTokenStream, ReplaceNodeTest, TestHashQParserPlugin, TestInPlaceUpdatesRequiredField, ConcurrentCreateRoutedAliasTest, TestFiltering, TestSolrIndexConfig, DirectoryFactoryTest, TestSolrJ, TestConfigSetsAPIExclusivity, SignificantTermsQParserPluginTest, ForceLeaderWithTlogReplicasTest, TestValueSourceCache, TestCloudPseudoReturnFields, TestFastOutputStream, TimeRoutedAliasUpdateProcessorTest, TestUnInvertedFieldException, TestSchemaManager, TestReloadAndDeleteDocs, CdcrWithNodesRestartsTest, DistribDocExpirationUpdateProcessorTest, TestFieldCacheSortRandom, TestStressInPlaceUpdates, AddReplicaTest, SortByFunctionTest, HttpPartitionOnCommitTest, PropertiesRequestHandlerTest, BadIndexSchemaTest, OverseerSolrResponseTest, HdfsTlogReplayBufferedWhileIndexingTest, RollingRestartTest, CoreAdminHandlerTest, SyncSliceTest, DistributedVersionInfoTest, ChaosMonkeyNothingIsSafeTest]
   [junit4] Completed [174/900 (1!)] on J1 in 43.03s, 1 test, 1 failure <<< FAILURES!

[...truncated 54901 lines...]