You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2017/07/25 15:08:43 UTC

[JENKINS] Lucene-Solr-7.x-Solaris (64bit/jdk1.8.0) - Build # 72 - Unstable!

Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Solaris/72/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseParallelGC

2 tests failed.
FAILED:  org.apache.solr.cloud.DocValuesNotIndexedTest.testGroupingSorting

Error Message:
Should have exactly 4 documents returned expected:<4> but was:<3>

Stack Trace:
java.lang.AssertionError: Should have exactly 4 documents returned expected:<4> but was:<3>
	at __randomizedtesting.SeedInfo.seed([64FBAC5A3BB96666:7AC3A4524712DCE6]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.apache.solr.cloud.DocValuesNotIndexedTest.checkSortOrder(DocValuesNotIndexedTest.java:260)
	at org.apache.solr.cloud.DocValuesNotIndexedTest.testGroupingSorting(DocValuesNotIndexedTest.java:245)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.client.solrj.io.stream.StreamExpressionTest.testExecutorStream

Error Message:
Error from server at https://127.0.0.1:58238/solr/mainCorpus_shard2_replica_n2: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 404 </title> </head> <body> <h2>HTTP ERROR: 404</h2> <p>Problem accessing /solr/mainCorpus_shard2_replica_n2/update. Reason: <pre>    Can not find: /solr/mainCorpus_shard2_replica_n2/update</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.14.v20161028</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.CloudSolrClient$RouteException: Error from server at https://127.0.0.1:58238/solr/mainCorpus_shard2_replica_n2: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 404 </title>
</head>
<body>
<h2>HTTP ERROR: 404</h2>
<p>Problem accessing /solr/mainCorpus_shard2_replica_n2/update. Reason:
<pre>    Can not find: /solr/mainCorpus_shard2_replica_n2/update</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.14.v20161028</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([4381740640705C3C:6141F5FD631A762C]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:539)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:993)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:862)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
	at org.apache.solr.client.solrj.request.UpdateRequest.commit(UpdateRequest.java:233)
	at org.apache.solr.client.solrj.io.stream.StreamExpressionTest.testExecutorStream(StreamExpressionTest.java:6750)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:58238/solr/mainCorpus_shard2_replica_n2: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 404 </title>
</head>
<body>
<h2>HTTP ERROR: 404</h2>
<p>Problem accessing /solr/mainCorpus_shard2_replica_n2/update. Reason:
<pre>    Can not find: /solr/mainCorpus_shard2_replica_n2/update</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.14.v20161028</a><hr/>
</body>
</html>

	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:588)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:252)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:241)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.lambda$directUpdate$0(CloudSolrClient.java:516)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	... 1 more




Build Log:
[...truncated 12458 lines...]
   [junit4] Suite: org.apache.solr.cloud.DocValuesNotIndexedTest
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.DocValuesNotIndexedTest_64FBAC5A3BB96666-001/init-core-data-001
   [junit4]   2> 1810964 WARN  (SUITE-DocValuesNotIndexedTest-seed#[64FBAC5A3BB96666]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=7 numCloses=7
   [junit4]   2> 1810965 INFO  (SUITE-DocValuesNotIndexedTest-seed#[64FBAC5A3BB96666]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 1810966 INFO  (SUITE-DocValuesNotIndexedTest-seed#[64FBAC5A3BB96666]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 1810966 INFO  (SUITE-DocValuesNotIndexedTest-seed#[64FBAC5A3BB96666]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 1810967 INFO  (SUITE-DocValuesNotIndexedTest-seed#[64FBAC5A3BB96666]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.DocValuesNotIndexedTest_64FBAC5A3BB96666-001/tempDir-001
   [junit4]   2> 1810967 INFO  (SUITE-DocValuesNotIndexedTest-seed#[64FBAC5A3BB96666]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1810968 INFO  (Thread-5162) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1810968 INFO  (Thread-5162) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1810988 ERROR (Thread-5162) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 1811067 INFO  (SUITE-DocValuesNotIndexedTest-seed#[64FBAC5A3BB96666]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:60326
   [junit4]   2> 1811095 INFO  (jetty-launcher-3992-thread-1) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 1811095 INFO  (jetty-launcher-3992-thread-2) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 1811097 INFO  (jetty-launcher-3992-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@675778c4{/solr,null,AVAILABLE}
   [junit4]   2> 1811097 INFO  (jetty-launcher-3992-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5810a062{/solr,null,AVAILABLE}
   [junit4]   2> 1811098 INFO  (jetty-launcher-3992-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@4fade0cf{HTTP/1.1,[http/1.1]}{127.0.0.1:39999}
   [junit4]   2> 1811098 INFO  (jetty-launcher-3992-thread-2) [    ] o.e.j.s.Server Started @1813704ms
   [junit4]   2> 1811098 INFO  (jetty-launcher-3992-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=39999}
   [junit4]   2> 1811099 ERROR (jetty-launcher-3992-thread-2) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1811099 INFO  (jetty-launcher-3992-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.1.0
   [junit4]   2> 1811099 INFO  (jetty-launcher-3992-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1811099 INFO  (jetty-launcher-3992-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 1811099 INFO  (jetty-launcher-3992-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-07-25T14:35:02.554Z
   [junit4]   2> 1811106 INFO  (jetty-launcher-3992-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@78e332d7{HTTP/1.1,[http/1.1]}{127.0.0.1:65238}
   [junit4]   2> 1811106 INFO  (jetty-launcher-3992-thread-1) [    ] o.e.j.s.Server Started @1813712ms
   [junit4]   2> 1811106 INFO  (jetty-launcher-3992-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=65238}
   [junit4]   2> 1811106 ERROR (jetty-launcher-3992-thread-1) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1811107 INFO  (jetty-launcher-3992-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.1.0
   [junit4]   2> 1811107 INFO  (jetty-launcher-3992-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1811107 INFO  (jetty-launcher-3992-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 1811107 INFO  (jetty-launcher-3992-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-07-25T14:35:02.562Z
   [junit4]   2> 1811109 INFO  (jetty-launcher-3992-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1811110 INFO  (jetty-launcher-3992-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1811118 INFO  (jetty-launcher-3992-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:60326/solr
   [junit4]   2> 1811124 INFO  (jetty-launcher-3992-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:60326/solr
   [junit4]   2> 1811251 INFO  (jetty-launcher-3992-thread-2) [n:127.0.0.1:39999_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1811251 INFO  (jetty-launcher-3992-thread-2) [n:127.0.0.1:39999_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:39999_solr
   [junit4]   2> 1811252 INFO  (jetty-launcher-3992-thread-2) [n:127.0.0.1:39999_solr    ] o.a.s.c.Overseer Overseer (id=98369097068904454-127.0.0.1:39999_solr-n_0000000000) starting
   [junit4]   2> 1811260 INFO  (jetty-launcher-3992-thread-2) [n:127.0.0.1:39999_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:39999_solr
   [junit4]   2> 1811263 INFO  (zkCallback-4004-thread-1-processing-n:127.0.0.1:39999_solr) [n:127.0.0.1:39999_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1811311 INFO  (jetty-launcher-3992-thread-1) [n:127.0.0.1:65238_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1811313 INFO  (jetty-launcher-3992-thread-1) [n:127.0.0.1:65238_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1811315 INFO  (jetty-launcher-3992-thread-1) [n:127.0.0.1:65238_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:65238_solr
   [junit4]   2> 1811316 INFO  (zkCallback-4004-thread-1-processing-n:127.0.0.1:39999_solr) [n:127.0.0.1:39999_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1811317 INFO  (zkCallback-4003-thread-1-processing-n:127.0.0.1:65238_solr) [n:127.0.0.1:65238_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1811400 INFO  (jetty-launcher-3992-thread-2) [n:127.0.0.1:39999_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_39999.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6e3c2249
   [junit4]   2> 1811405 INFO  (jetty-launcher-3992-thread-2) [n:127.0.0.1:39999_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_39999.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6e3c2249
   [junit4]   2> 1811405 INFO  (jetty-launcher-3992-thread-2) [n:127.0.0.1:39999_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_39999.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6e3c2249
   [junit4]   2> 1811406 INFO  (jetty-launcher-3992-thread-2) [n:127.0.0.1:39999_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.DocValuesNotIndexedTest_64FBAC5A3BB96666-001/tempDir-001/node2/.
   [junit4]   2> 1811448 INFO  (jetty-launcher-3992-thread-1) [n:127.0.0.1:65238_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_65238.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6e3c2249
   [junit4]   2> 1811450 INFO  (jetty-launcher-3992-thread-1) [n:127.0.0.1:65238_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_65238.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6e3c2249
   [junit4]   2> 1811450 INFO  (jetty-launcher-3992-thread-1) [n:127.0.0.1:65238_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_65238.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6e3c2249
   [junit4]   2> 1811450 INFO  (jetty-launcher-3992-thread-1) [n:127.0.0.1:65238_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.DocValuesNotIndexedTest_64FBAC5A3BB96666-001/tempDir-001/node1/.
   [junit4]   2> 1811469 INFO  (SUITE-DocValuesNotIndexedTest-seed#[64FBAC5A3BB96666]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1811471 INFO  (SUITE-DocValuesNotIndexedTest-seed#[64FBAC5A3BB96666]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:60326/solr ready
   [junit4]   2> 1811489 INFO  (qtp2106332852-21936) [n:127.0.0.1:65238_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&maxShardsPerNode=2&name=dv_coll&nrtReplicas=1&action=CREATE&numShards=4&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1811492 INFO  (OverseerThreadFactory-6864-thread-1-processing-n:127.0.0.1:39999_solr) [n:127.0.0.1:39999_solr    ] o.a.s.c.CreateCollectionCmd Create collection dv_coll
   [junit4]   2> 1811643 INFO  (OverseerStateUpdate-98369097068904454-127.0.0.1:39999_solr-n_0000000000) [n:127.0.0.1:39999_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"dv_coll",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"dv_coll_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:65238/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 1811655 INFO  (OverseerStateUpdate-98369097068904454-127.0.0.1:39999_solr-n_0000000000) [n:127.0.0.1:39999_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"dv_coll",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"dv_coll_shard2_replica_n2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:39999/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 1811659 INFO  (OverseerStateUpdate-98369097068904454-127.0.0.1:39999_solr-n_0000000000) [n:127.0.0.1:39999_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"dv_coll",
   [junit4]   2>   "shard":"shard3",
   [junit4]   2>   "core":"dv_coll_shard3_replica_n4",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:65238/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 1811662 INFO  (OverseerStateUpdate-98369097068904454-127.0.0.1:39999_solr-n_0000000000) [n:127.0.0.1:39999_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"dv_coll",
   [junit4]   2>   "shard":"shard4",
   [junit4]   2>   "core":"dv_coll_shard4_replica_n5",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:39999/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 1811872 INFO  (qtp2108814188-21928) [n:127.0.0.1:39999_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node8&collection.configName=conf1&newCollection=true&name=dv_coll_shard4_replica_n5&action=CREATE&numShards=4&collection=dv_coll&shard=shard4&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1811872 INFO  (qtp2106332852-21938) [n:127.0.0.1:65238_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf1&newCollection=true&name=dv_coll_shard1_replica_n1&action=CREATE&numShards=4&collection=dv_coll&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1811872 INFO  (qtp2108814188-21928) [n:127.0.0.1:39999_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1811872 INFO  (qtp2106332852-21938) [n:127.0.0.1:65238_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1811873 INFO  (qtp2108814188-21933) [n:127.0.0.1:39999_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node6&collection.configName=conf1&newCollection=true&name=dv_coll_shard2_replica_n2&action=CREATE&numShards=4&collection=dv_coll&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1811874 INFO  (qtp2106332852-21937) [n:127.0.0.1:65238_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node7&collection.configName=conf1&newCollection=true&name=dv_coll_shard3_replica_n4&action=CREATE&numShards=4&collection=dv_coll&shard=shard3&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1812023 INFO  (zkCallback-4003-thread-1-processing-n:127.0.0.1:65238_solr) [n:127.0.0.1:65238_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1812024 INFO  (zkCallback-4003-thread-2-processing-n:127.0.0.1:65238_solr) [n:127.0.0.1:65238_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1812029 INFO  (zkCallback-4004-thread-1-processing-n:127.0.0.1:39999_solr) [n:127.0.0.1:39999_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1812029 INFO  (zkCallback-4004-thread-2-processing-n:127.0.0.1:39999_solr) [n:127.0.0.1:39999_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1812913 INFO  (qtp2106332852-21938) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 1812914 INFO  (qtp2106332852-21937) [n:127.0.0.1:65238_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 1812919 INFO  (qtp2108814188-21933) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 1812920 INFO  (qtp2106332852-21938) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.IndexSchema [dv_coll_shard1_replica_n1] Schema name=minimal
   [junit4]   2> 1812922 INFO  (qtp2106332852-21937) [n:127.0.0.1:65238_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.s.IndexSchema [dv_coll_shard3_replica_n4] Schema name=minimal
   [junit4]   2> 1812923 INFO  (qtp2106332852-21938) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1812923 INFO  (qtp2106332852-21938) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'dv_coll_shard1_replica_n1' using configuration from collection dv_coll, trusted=true
   [junit4]   2> 1812923 INFO  (qtp2106332852-21938) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_65238.solr.core.dv_coll.shard1.replica_n1' (registry 'solr.core.dv_coll.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6e3c2249
   [junit4]   2> 1812923 INFO  (qtp2106332852-21938) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1812923 INFO  (qtp2106332852-21938) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.SolrCore [[dv_coll_shard1_replica_n1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.DocValuesNotIndexedTest_64FBAC5A3BB96666-001/tempDir-001/node1/dv_coll_shard1_replica_n1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.DocValuesNotIndexedTest_64FBAC5A3BB96666-001/tempDir-001/node1/./dv_coll_shard1_replica_n1/data/]
   [junit4]   2> 1812924 INFO  (qtp2106332852-21937) [n:127.0.0.1:65238_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1812924 INFO  (qtp2106332852-21937) [n:127.0.0.1:65238_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.c.CoreContainer Creating SolrCore 'dv_coll_shard3_replica_n4' using configuration from collection dv_coll, trusted=true
   [junit4]   2> 1812924 INFO  (qtp2108814188-21928) [n:127.0.0.1:39999_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n5] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 1812924 INFO  (qtp2106332852-21937) [n:127.0.0.1:65238_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_65238.solr.core.dv_coll.shard3.replica_n4' (registry 'solr.core.dv_coll.shard3.replica_n4') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6e3c2249
   [junit4]   2> 1812925 INFO  (qtp2106332852-21937) [n:127.0.0.1:65238_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1812925 INFO  (qtp2106332852-21937) [n:127.0.0.1:65238_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.c.SolrCore [[dv_coll_shard3_replica_n4] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.DocValuesNotIndexedTest_64FBAC5A3BB96666-001/tempDir-001/node1/dv_coll_shard3_replica_n4], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.DocValuesNotIndexedTest_64FBAC5A3BB96666-001/tempDir-001/node1/./dv_coll_shard3_replica_n4/data/]
   [junit4]   2> 1812929 INFO  (qtp2108814188-21928) [n:127.0.0.1:39999_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n5] o.a.s.s.IndexSchema [dv_coll_shard4_replica_n5] Schema name=minimal
   [junit4]   2> 1812932 INFO  (qtp2108814188-21933) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.s.IndexSchema [dv_coll_shard2_replica_n2] Schema name=minimal
   [junit4]   2> 1812933 INFO  (qtp2108814188-21928) [n:127.0.0.1:39999_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n5] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1812933 INFO  (qtp2108814188-21928) [n:127.0.0.1:39999_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n5] o.a.s.c.CoreContainer Creating SolrCore 'dv_coll_shard4_replica_n5' using configuration from collection dv_coll, trusted=true
   [junit4]   2> 1812934 INFO  (qtp2108814188-21928) [n:127.0.0.1:39999_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n5] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_39999.solr.core.dv_coll.shard4.replica_n5' (registry 'solr.core.dv_coll.shard4.replica_n5') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6e3c2249
   [junit4]   2> 1812934 INFO  (qtp2108814188-21928) [n:127.0.0.1:39999_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n5] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1812934 INFO  (qtp2108814188-21928) [n:127.0.0.1:39999_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n5] o.a.s.c.SolrCore [[dv_coll_shard4_replica_n5] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.DocValuesNotIndexedTest_64FBAC5A3BB96666-001/tempDir-001/node2/dv_coll_shard4_replica_n5], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.DocValuesNotIndexedTest_64FBAC5A3BB96666-001/tempDir-001/node2/./dv_coll_shard4_replica_n5/data/]
   [junit4]   2> 1812937 INFO  (qtp2108814188-21933) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1812937 INFO  (qtp2108814188-21933) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.c.CoreContainer Creating SolrCore 'dv_coll_shard2_replica_n2' using configuration from collection dv_coll, trusted=true
   [junit4]   2> 1812938 INFO  (qtp2108814188-21933) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_39999.solr.core.dv_coll.shard2.replica_n2' (registry 'solr.core.dv_coll.shard2.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6e3c2249
   [junit4]   2> 1812938 INFO  (qtp2108814188-21933) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1812938 INFO  (qtp2108814188-21933) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.c.SolrCore [[dv_coll_shard2_replica_n2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.DocValuesNotIndexedTest_64FBAC5A3BB96666-001/tempDir-001/node2/dv_coll_shard2_replica_n2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.DocValuesNotIndexedTest_64FBAC5A3BB96666-001/tempDir-001/node2/./dv_coll_shard2_replica_n2/data/]
   [junit4]   2> 1813041 INFO  (qtp2106332852-21937) [n:127.0.0.1:65238_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1813042 INFO  (qtp2106332852-21937) [n:127.0.0.1:65238_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1813043 INFO  (qtp2106332852-21937) [n:127.0.0.1:65238_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1813043 INFO  (qtp2106332852-21937) [n:127.0.0.1:65238_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1813048 INFO  (qtp2106332852-21937) [n:127.0.0.1:65238_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.s.SolrIndexSearcher Opening [Searcher@3d3b5246[dv_coll_shard3_replica_n4] main]
   [junit4]   2> 1813049 INFO  (qtp2106332852-21937) [n:127.0.0.1:65238_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1813050 INFO  (qtp2106332852-21937) [n:127.0.0.1:65238_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1813050 INFO  (qtp2106332852-21937) [n:127.0.0.1:65238_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/conf1/managed-schema
   [junit4]   2> 1813050 INFO  (qtp2108814188-21933) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1813050 INFO  (qtp2108814188-21933) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1813051 INFO  (qtp2106332852-21937) [n:127.0.0.1:65238_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
   [junit4]   2> 1813056 INFO  (qtp2106332852-21937) [n:127.0.0.1:65238_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1813056 INFO  (qtp2106332852-21937) [n:127.0.0.1:65238_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1573905555270926336
   [junit4]   2> 1813056 INFO  (qtp2108814188-21933) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1813056 INFO  (qtp2108814188-21933) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1813057 INFO  (qtp2106332852-21938) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1813057 INFO  (qtp2106332852-21938) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1813060 INFO  (searcherExecutor-6870-thread-1-processing-n:127.0.0.1:65238_solr x:dv_coll_shard3_replica_n4 s:shard3 c:dv_coll r:core_node7) [n:127.0.0.1:65238_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.c.SolrCore [dv_coll_shard3_replica_n4] Registered new searcher Searcher@3d3b5246[dv_coll_shard3_replica_n4] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1813060 INFO  (qtp2106332852-21938) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1813060 INFO  (qtp2106332852-21938) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1813061 INFO  (qtp2108814188-21933) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@7650a33a[dv_coll_shard2_replica_n2] main]
   [junit4]   2> 1813063 INFO  (qtp2108814188-21933) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1813063 INFO  (qtp2106332852-21938) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@34535ae[dv_coll_shard1_replica_n1] main]
   [junit4]   2> 1813063 INFO  (qtp2108814188-21933) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1813064 INFO  (qtp2108814188-21933) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/conf1/managed-schema
   [junit4]   2> 1813065 INFO  (qtp2108814188-21933) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
   [junit4]   2> 1813066 INFO  (qtp2106332852-21937) [n:127.0.0.1:65238_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1813066 INFO  (qtp2106332852-21937) [n:127.0.0.1:65238_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1813066 INFO  (qtp2106332852-21937) [n:127.0.0.1:65238_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:65238/solr/dv_coll_shard3_replica_n4/
   [junit4]   2> 1813066 INFO  (qtp2106332852-21937) [n:127.0.0.1:65238_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 1813066 INFO  (qtp2106332852-21937) [n:127.0.0.1:65238_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.c.SyncStrategy http://127.0.0.1:65238/solr/dv_coll_shard3_replica_n4/ has no replicas
   [junit4]   2> 1813067 INFO  (qtp2106332852-21937) [n:127.0.0.1:65238_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 1813066 INFO  (qtp2106332852-21938) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1813066 INFO  (qtp2108814188-21933) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1813068 INFO  (qtp2106332852-21938) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1813068 INFO  (qtp2106332852-21938) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/conf1/managed-schema
   [junit4]   2> 1813068 INFO  (searcherExecutor-6872-thread-1-processing-n:127.0.0.1:39999_solr x:dv_coll_shard2_replica_n2 s:shard2 c:dv_coll r:core_node6) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.c.SolrCore [dv_coll_shard2_replica_n2] Registered new searcher Searcher@7650a33a[dv_coll_shard2_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1813068 INFO  (qtp2108814188-21933) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1573905555283509248
   [junit4]   2> 1813069 INFO  (qtp2106332852-21938) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
   [junit4]   2> 1813069 INFO  (qtp2108814188-21928) [n:127.0.0.1:39999_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n5] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1813069 INFO  (qtp2108814188-21928) [n:127.0.0.1:39999_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n5] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1813070 INFO  (qtp2106332852-21938) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1813070 INFO  (qtp2106332852-21938) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1573905555285606400
   [junit4]   2> 1813070 INFO  (searcherExecutor-6869-thread-1-processing-n:127.0.0.1:65238_solr x:dv_coll_shard1_replica_n1 s:shard1 c:dv_coll r:core_node3) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.SolrCore [dv_coll_shard1_replica_n1] Registered new searcher Searcher@34535ae[dv_coll_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1813071 INFO  (qtp2108814188-21928) [n:127.0.0.1:39999_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n5] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1813071 INFO  (qtp2108814188-21928) [n:127.0.0.1:39999_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n5] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1813074 INFO  (qtp2108814188-21928) [n:127.0.0.1:39999_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n5] o.a.s.s.SolrIndexSearcher Opening [Searcher@63fe6a6b[dv_coll_shard4_replica_n5] main]
   [junit4]   2> 1813076 INFO  (qtp2108814188-21928) [n:127.0.0.1:39999_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n5] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1813076 INFO  (qtp2108814188-21928) [n:127.0.0.1:39999_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n5] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1813080 INFO  (qtp2108814188-21928) [n:127.0.0.1:39999_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n5] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/conf1/managed-schema
   [junit4]   2> 1813080 INFO  (qtp2106332852-21937) [n:127.0.0.1:65238_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:65238/solr/dv_coll_shard3_replica_n4/ shard3
   [junit4]   2> 1813080 INFO  (qtp2108814188-21933) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1813081 INFO  (qtp2108814188-21933) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1813081 INFO  (qtp2108814188-21933) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:39999/solr/dv_coll_shard2_replica_n2/
   [junit4]   2> 1813081 INFO  (qtp2108814188-21933) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 1813081 INFO  (qtp2108814188-21928) [n:127.0.0.1:39999_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n5] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
   [junit4]   2> 1813081 INFO  (qtp2108814188-21933) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.c.SyncStrategy http://127.0.0.1:39999/solr/dv_coll_shard2_replica_n2/ has no replicas
   [junit4]   2> 1813081 INFO  (qtp2108814188-21933) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 1813083 INFO  (qtp2108814188-21928) [n:127.0.0.1:39999_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n5] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1813083 INFO  (qtp2108814188-21928) [n:127.0.0.1:39999_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n5] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1573905555299237888
   [junit4]   2> 1813085 INFO  (qtp2106332852-21938) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1813086 INFO  (qtp2106332852-21938) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1813086 INFO  (qtp2106332852-21938) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:65238/solr/dv_coll_shard1_replica_n1/
   [junit4]   2> 1813086 INFO  (qtp2106332852-21938) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 1813086 INFO  (qtp2106332852-21938) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:65238/solr/dv_coll_shard1_replica_n1/ has no replicas
   [junit4]   2> 1813086 INFO  (qtp2106332852-21938) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 1813087 INFO  (searcherExecutor-6871-thread-1-processing-n:127.0.0.1:39999_solr x:dv_coll_shard4_replica_n5 s:shard4 c:dv_coll r:core_node8) [n:127.0.0.1:39999_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n5] o.a.s.c.SolrCore [dv_coll_shard4_replica_n5] Registered new searcher Searcher@63fe6a6b[dv_coll_shard4_replica_n5] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1813092 INFO  (qtp2106332852-21938) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:65238/solr/dv_coll_shard1_replica_n1/ shard1
   [junit4]   2> 1813095 INFO  (qtp2108814188-21933) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:39999/solr/dv_coll_shard2_replica_n2/ shard2
   [junit4]   2> 1813099 INFO  (qtp2108814188-21928) [n:127.0.0.1:39999_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n5] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1813099 INFO  (qtp2108814188-21928) [n:127.0.0.1:39999_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n5] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1813099 INFO  (qtp2108814188-21928) [n:127.0.0.1:39999_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n5] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:39999/solr/dv_coll_shard4_replica_n5/
   [junit4]   2> 1813099 INFO  (qtp2108814188-21928) [n:127.0.0.1:39999_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n5] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 1813100 INFO  (qtp2108814188-21928) [n:127.0.0.1:39999_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n5] o.a.s.c.SyncStrategy http://127.0.0.1:39999/solr/dv_coll_shard4_replica_n5/ has no replicas
   [junit4]   2> 1813100 INFO  (qtp2108814188-21928) [n:127.0.0.1:39999_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n5] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 1813104 INFO  (qtp2108814188-21928) [n:127.0.0.1:39999_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n5] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:39999/solr/dv_coll_shard4_replica_n5/ shard4
   [junit4]   2> 1813208 INFO  (zkCallback-4004-thread-2-processing-n:127.0.0.1:39999_solr) [n:127.0.0.1:39999_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1813208 INFO  (zkCallback-4004-thread-1-processing-n:127.0.0.1:39999_solr) [n:127.0.0.1:39999_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1813209 INFO  (zkCallback-4003-thread-1-processing-n:127.0.0.1:65238_solr) [n:127.0.0.1:65238_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1813209 INFO  (zkCallback-4003-thread-2-processing-n:127.0.0.1:65238_solr) [n:127.0.0.1:65238_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1813235 INFO  (qtp2106332852-21937) [n:127.0.0.1:65238_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1813237 INFO  (qtp2106332852-21937) [n:127.0.0.1:65238_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node7&collection.configName=conf1&newCollection=true&name=dv_coll_shard3_replica_n4&action=CREATE&numShards=4&collection=dv_coll&shard=shard3&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1364
   [junit4]   2> 1813244 INFO  (qtp2106332852-21938) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1813250 INFO  (qtp2106332852-21938) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf1&newCollection=true&name=dv_coll_shard1_replica_n1&action=CREATE&numShards=4&collection=dv_coll&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1378
   [junit4]   2> 1813254 INFO  (qtp2108814188-21933) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1813255 INFO  (qtp2108814188-21928) [n:127.0.0.1:39999_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n5] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1813256 INFO  (qtp2108814188-21933) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node6&collection.configName=conf1&newCollection=true&name=dv_coll_shard2_replica_n2&action=CREATE&numShards=4&collection=dv_coll&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1384
   [junit4]   2> 1813258 INFO  (qtp2108814188-21928) [n:127.0.0.1:39999_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n5] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node8&collection.configName=conf1&newCollection=true&name=dv_coll_shard4_replica_n5&action=CREATE&numShards=4&collection=dv_coll&shard=shard4&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1386
   [junit4]   2> 1813264 INFO  (qtp2106332852-21936) [n:127.0.0.1:65238_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 1813365 INFO  (zkCallback-4004-thread-1-processing-n:127.0.0.1:39999_solr) [n:127.0.0.1:39999_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1813365 INFO  (zkCallback-4004-thread-2-processing-n:127.0.0.1:39999_solr) [n:127.0.0.1:39999_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1813366 INFO  (zkCallback-4003-thread-2-processing-n:127.0.0.1:65238_solr) [n:127.0.0.1:65238_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1813366 INFO  (zkCallback-4003-thread-1-processing-n:127.0.0.1:65238_solr) [n:127.0.0.1:65238_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1813498 INFO  (OverseerCollectionConfigSetProcessor-98369097068904454-127.0.0.1:39999_solr-n_0000000000) [n:127.0.0.1:39999_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1814267 INFO  (qtp2106332852-21936) [n:127.0.0.1:65238_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&maxShardsPerNode=2&name=dv_coll&nrtReplicas=1&action=CREATE&numShards=4&wt=javabin&version=2} status=0 QTime=2778
   [junit4]   2> 1814272 INFO  (qtp2106332852-21940) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.SchemaManager managed schema loaded . version : 0 
   [junit4]   2> 1814274 INFO  (qtp2106332852-21940) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.IndexSchema [dv_coll_shard1_replica_n1] Schema name=minimal
   [junit4]   2> 1814276 INFO  (qtp2106332852-21940) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1814284 INFO  (qtp2106332852-21940) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.ZkController Persisted config data to node /configs/conf1/managed-schema 
   [junit4]   2> 1814284 INFO  (zkCallback-4004-thread-2-processing-n:127.0.0.1:39999_solr) [n:127.0.0.1:39999_solr    ] o.a.s.s.ZkIndexSchemaReader A schema change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/configs/conf1/managed-schema, has occurred - updating schema from ZooKeeper ...
   [junit4]   2> 1814284 INFO  (zkCallback-4003-thread-2-processing-n:127.0.0.1:65238_solr) [n:127.0.0.1:65238_solr    ] o.a.s.s.ZkIndexSchemaReader A schema change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/configs/conf1/managed-schema, has occurred - updating schema from ZooKeeper ...
   [junit4]   2> 1814284 INFO  (zkCallback-4004-thread-1-processing-n:127.0.0.1:39999_solr) [n:127.0.0.1:39999_solr    ] o.a.s.s.ZkIndexSchemaReader A schema change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/configs/conf1/managed-schema, has occurred - updating schema from ZooKeeper ...
   [junit4]   2> 1814285 INFO  (zkCallback-4003-thread-1-processing-n:127.0.0.1:65238_solr) [n:127.0.0.1:65238_solr    ] o.a.s.s.ZkIndexSchemaReader A schema change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/configs/conf1/managed-schema, has occurred - updating schema from ZooKeeper ...
   [junit4]   2> 1814285 INFO  (zkCallback-4003-thread-2-processing-n:127.0.0.1:65238_solr) [n:127.0.0.1:65238_solr    ] o.a.s.s.ZkIndexSchemaReader Retrieved schema version 1 from ZooKeeper
   [junit4]   2> 1814285 INFO  (zkCallback-4004-thread-2-processing-n:127.0.0.1:39999_solr) [n:127.0.0.1:39999_solr    ] o.a.s.s.ZkIndexSchemaReader Retrieved schema version 1 from ZooKeeper
   [junit4]   2> 1814288 INFO  (zkCallback-4003-thread-2-processing-n:127.0.0.1:65238_solr) [n:127.0.0.1:65238_solr    ] o.a.s.s.IndexSchema [dv_coll_shard3_replica_n4] Schema name=minimal
   [junit4]   2> 1814288 INFO  (zkCallback-4004-thread-2-processing-n:127.0.0.1:39999_solr) [n:127.0.0.1:39999_solr    ] o.a.s.s.IndexSchema [dv_coll_shard4_replica_n5] Schema name=minimal
   [junit4]   2> 1814292 INFO  (zkCallback-4003-thread-2-processing-n:127.0.0.1:65238_solr) [n:127.0.0.1:65238_solr    ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1814292 INFO  (zkCallback-4003-thread-2-processing-n:127.0.0.1:65238_solr) [n:127.0.0.1:65238_solr    ] o.a.s.s.ZkIndexSchemaReader Finished refreshing schema in 7 ms
   [junit4]   2> 1814292 INFO  (zkCallback-4004-thread-2-processing-n:127.0.0.1:39999_solr) [n:127.0.0.1:39999_solr    ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1814292 INFO  (zkCallback-4004-thread-2-processing-n:127.0.0.1:39999_solr) [n:127.0.0.1:39999_solr    ] o.a.s.s.ZkIndexSchemaReader Finished refreshing schema in 7 ms
   [junit4]   2> 1814293 INFO  (Thread-5184) [n:127.0.0.1:65238_solr    ] o.a.s.c.SolrCore config update listener called for core dv_coll_shard3_replica_n4
   [junit4]   2> 1814297 INFO  (qtp2106332852-21940) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 1814302 INFO  (zkCallback-4004-thread-1-processing-n:127.0.0.1:39999_solr) [n:127.0.0.1:39999_solr    ] o.a.s.s.ZkIndexSchemaReader Retrieved schema version 1 from ZooKeeper
   [junit4]   2> 1814302 INFO  (Thread-5185) [n:127.0.0.1:39999_solr    ] o.a.s.c.SolrCore config update listener called for core dv_coll_shard2_replica_n2
   [junit4]   2> 1814305 INFO  (zkCallback-4004-thread-1-processing-n:127.0.0.1:39999_solr) [n:127.0.0.1:39999_solr    ] o.a.s.s.IndexSchema [dv_coll_shard2_replica_n2] Schema name=minimal
   [junit4]   2> 1814335 INFO  (zkCallback-4004-thread-1-processing-n:127.0.0.1:39999_solr) [n:127.0.0.1:39999_solr    ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1814336 INFO  (zkCallback-4004-thread-1-processing-n:127.0.0.1:39999_solr) [n:127.0.0.1:39999_solr    ] o.a.s.s.ZkIndexSchemaReader Finished refreshing schema in 33 ms
   [junit4]   2> 1814337 INFO  (Thread-5184) [n:127.0.0.1:65238_solr    ] o.a.s.c.SolrCore config update listener called for core dv_coll_shard1_replica_n1
   [junit4]   2> 1814338 INFO  (Thread-5185) [n:127.0.0.1:39999_solr    ] o.a.s.c.SolrCore core reload dv_coll_shard2_replica_n2
   [junit4]   2> 1814342 INFO  (qtp2106332852-21940) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.IndexSchema [dv_coll_shard1_replica_n1] Schema name=minimal
   [junit4]   2> 1814346 INFO  (qtp2106332852-21940) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1814346 INFO  (qtp2106332852-21940) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.CoreContainer Reloading SolrCore 'dv_coll_shard1_replica_n1' using configuration from collection dv_coll
   [junit4]   2> 1814346 INFO  (Thread-5185) [n:127.0.0.1:39999_solr    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 1814357 INFO  (Thread-5185) [n:127.0.0.1:39999_solr    ] o.a.s.s.IndexSchema [dv_coll_shard2_replica_n2] Schema name=minimal
   [junit4]   2> 1814359 INFO  (qtp2106332852-21940) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_65238.solr.core.dv_coll.shard1.replica_n1' (registry 'solr.core.dv_coll.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6e3c2249
   [junit4]   2> 1814359 INFO  (qtp2106332852-21940) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.SolrCore [[dv_coll_shard1_replica_n1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.DocValuesNotIndexedTest_64FBAC5A3BB96666-001/tempDir-001/node1/dv_coll_shard1_replica_n1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.DocValuesNotIndexedTest_64FBAC5A3BB96666-001/tempDir-001/node1/./dv_coll_shard1_replica_n1/data/]
   [junit4]   2> 1814361 INFO  (Thread-5185) [n:127.0.0.1:39999_solr    ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1814361 INFO  (Thread-5185) [n:127.0.0.1:39999_solr    ] o.a.s.c.CoreContainer Reloading SolrCore 'dv_coll_shard2_replica_n2' using configuration from collection dv_coll
   [junit4]   2> 1814369 INFO  (Thread-5185) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_39999.solr.core.dv_coll.shard2.replica_n2' (registry 'solr.core.dv_coll.shard2.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6e3c2249
   [junit4]   2> 1814369 INFO  (Thread-5185) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.c.SolrCore [[dv_coll_shard2_replica_n2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.DocValuesNotIndexedTest_64FBAC5A3BB96666-001/tempDir-001/node2/dv_coll_shard2_replica_n2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.DocValuesNotIndexedTest_64FBAC5A3BB96666-001/tempDir-001/node2/./dv_coll_shard2_replica_n2/data/]
   [junit4]   2> 1814382 INFO  (Thread-5184) [n:127.0.0.1:65238_solr    ] o.a.s.c.SolrCore core reload dv_coll_shard1_replica_n1
   [junit4]   2> 1814436 INFO  (Thread-5185) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1814436 INFO  (Thread-5185) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1814437 INFO  (Thread-5185) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@5ccb0a8b[dv_coll_shard2_replica_n2] main]
   [junit4]   2> 1814438 INFO  (qtp2106332852-21940) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1814438 INFO  (qtp2106332852-21940) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1814439 INFO  (Thread-5185) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1814439 INFO  (Thread-5185) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1814439 INFO  (Thread-5185) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/conf1/managed-schema
   [junit4]   2> 1814439 INFO  (qtp2106332852-21940) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@4ecaa41d[dv_coll_shard1_replica_n1] main]
   [junit4]   2> 1814441 INFO  (qtp2106332852-21940) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1814442 INFO  (qtp2106332852-21940) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1814442 INFO  (qtp2106332852-21940) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/conf1/managed-schema
   [junit4]   2> 1814443 INFO  (qtp2106332852-21940) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.ZkIndexSchemaReader Current schema version 1 is already the latest
   [junit4]   2> 1814443 INFO  (Thread-5185) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.s.ZkIndexSchemaReader Current schema version 1 is already the latest
   [junit4]   2> 1814443 INFO  (Thread-5185) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1814443 INFO  (qtp2106332852-21940) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1814445 INFO  (searcherExecutor-6889-thread-1-processing-n:127.0.0.1:65238_solr x:dv_coll_shard1_replica_n1 s:shard1 c:dv_coll r:core_node3) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.SolrCore [dv_coll_shard1_replica_n1] Registered new searcher Searcher@4ecaa41d[dv_coll_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1814445 INFO  (searcherExecutor-6890-thread-1-processing-n:127.0.0.1:39999_solr x:dv_coll_shard2_replica_n2 s:shard2 c:dv_coll r:core_node6) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.c.SolrCore [dv_coll_shard2_replica_n2] Registered new searcher Searcher@5ccb0a8b[dv_coll_shard2_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1814450 INFO  (qtp2106332852-21940) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1573905556732641280
   [junit4]   2> 1814454 INFO  (qtp2106332852-21940) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
   [junit4]   2> 1814454 INFO  (qtp2106332852-21940) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@533d1422[dv_coll_shard1_replica_n1] main]
   [junit4]   2> 1814456 INFO  (Thread-5185) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1573905556738932736
   [junit4]   2> 1814457 INFO  (qtp2106332852-21940) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.ManagedIndexSchema Waiting up to 599 secs for 3 replicas to apply schema update version 1 for collection dv_coll
   [junit4]   2> 1814458 INFO  (searcherExecutor-6889-thread-1-processing-n:127.0.0.1:65238_solr x:dv_coll_shard1_replica_n1 s:shard1 c:dv_coll r:core_node3) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.SolrCore [dv_coll_shard1_replica_n1] Registered new searcher Searcher@533d1422[dv_coll_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1814464 INFO  (Thread-5185) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
   [junit4]   2> 1814464 INFO  (Thread-5185) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@2550b46c[dv_coll_shard2_replica_n2] main]
   [junit4]   2> 1814465 INFO  (Thread-5185) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.c.SolrCore [dv_coll_shard2_replica_n2]  CLOSING SolrCore org.apache.solr.core.SolrCore@15fb545f
   [junit4]   2> 1814467 INFO  (searcherExecutor-6890-thread-1-processing-n:127.0.0.1:39999_solr x:dv_coll_shard2_replica_n2 s:shard2 c:dv_coll r:core_node6) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.c.SolrCore [dv_coll_shard2_replica_n2] Registered new searcher Searcher@2550b46c[dv_coll_shard2_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1814467 INFO  (Thread-5185) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.dv_coll.shard2.replica_n2, tag=368792671
   [junit4]   2> 1814470 INFO  (qtp2106332852-21944) [n:127.0.0.1:65238_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.c.S.Request [dv_coll_shard3_replica_n4]  webapp=/solr path=/schema/zkversion params={refreshIfBelowVersion=1&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1814472 INFO  (Thread-5185) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.dv_coll.shard2.leader, tag=368792671
   [junit4]   2> 1814472 INFO  (Thread-5185) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.c.SolrCore config update listener called for core dv_coll_shard4_replica_n5
   [junit4]   2> 1814474 INFO  (qtp2108814188-21934) [n:127.0.0.1:39999_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n5] o.a.s.c.S.Request [dv_coll_shard4_replica_n5]  webapp=/solr path=/schema/zkversion params={refreshIfBelowVersion=1&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1814475 INFO  (zkCallback-4003-thread-1-processing-n:127.0.0.1:65238_solr) [n:127.0.0.1:65238_solr    ] o.a.s.s.ZkIndexSchemaReader Retrieved schema version 1 from ZooKeeper
   [junit4]   2> 1814479 INFO  (zkCallback-4003-thread-1-processing-n:127.0.0.1:65238_solr) [n:127.0.0.1:65238_solr    ] o.a.s.s.IndexSchema [dv_coll_shard1_replica_n1] Schema name=minimal
   [junit4]   2> 1814484 INFO  (Thread-5184) [n:127.0.0.1:65238_solr    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 1814487 INFO  (qtp2108814188-21931) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.c.S.Request [dv_coll_shard2_replica_n2]  webapp=/solr path=/schema/zkversion params={refreshIfBelowVersion=1&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1814488 INFO  (zkCallback-4003-thread-1-processing-n:127.0.0.1:65238_solr) [n:127.0.0.1:65238_solr    ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1814488 INFO  (zkCallback-4003-thread-1-processing-n:127.0.0.1:65238_solr) [n:127.0.0.1:65238_solr    ] o.a.s.s.ZkIndexSchemaReader Finished refreshing schema in 12 ms
   [junit4]   2> 1814494 INFO  (qtp2106332852-21940) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.ManagedIndexSchema Took 37.0ms for 3 replicas to apply schema update version 1 for collection dv_coll
   [junit4]   2> 1814494 INFO  (qtp2106332852-21940) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.S.Request [dv_coll_shard1_replica_n1]  webapp=/solr path=/schema params={wt=javabin&version=2} status=0 QTime=223
   [junit4]   2> 1814494 INFO  (qtp2106332852-21940) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.SolrCore [dv_coll_shard1_replica_n1]  CLOSING SolrCore org.apache.solr.core.SolrCore@edd0160
   [junit4]   2> 1814495 INFO  (qtp2106332852-21940) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.dv_coll.shard1.replica_n1, tag=249364832
   [junit4]   2> 1814495 INFO  (Thread-5184) [n:127.0.0.1:65238_solr    ] o.a.s.s.IndexSchema [dv_coll_shard1_replica_n1] Schema name=minimal
   [junit4]   2> 1814522 INFO  (Thread-5184) [n:127.0.0.1:65238_solr    ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1814522 INFO  (Thread-5184) [n:127.0.0.1:65238_solr    ] o.a.s.c.CoreContainer Reloading SolrCore 'dv_coll_shard1_replica_n1' using configuration from collection dv_coll
   [junit4]   2> 1814523 INFO  (qtp2106332852-21940) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.dv_coll.shard1.leader, tag=249364832
   [junit4]   2> 1814723 INFO  (TEST-DocValuesNotIndexedTest.testGroupingSorting-seed#[64FBAC5A3BB96666]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testGroupingSorting
   [junit4]   2> 1814735 INFO  (Thread-5184) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_65238.solr.core.dv_coll.shard1.replica_n1' (registry 'solr.core.dv_coll.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6e3c2249
   [junit4]   2> 1814735 INFO  (Thread-5184) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.SolrCore [[dv_coll_shard1_replica_n1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.DocValuesNotIndexedTest_64FBAC5A3BB96666-001/tempDir-001/node1/dv_coll_shard1_replica_n1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.DocValuesNotIndexedTest_64FBAC5A3BB96666-001/tempDir-001/node1/./dv_coll_shard1_replica_n1/data/]
   [junit4]   2> 1814735 INFO  (qtp2108814188-21932) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard2_replica_n2]  webapp=/solr path=/update params={update.distrib=TOLEADER&distrib.from=http://127.0.0.1:65238/solr/dv_coll_shard3_replica_n4/&wt=javabin&version=2}{deleteByQuery=*:* (-1573905557020999680)} 0 10
   [junit4]   2> 1814736 INFO  (qtp2108814188-21934) [n:127.0.0.1:39999_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n5] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard4_replica_n5]  webapp=/solr path=/update params={update.distrib=TOLEADER&distrib.from=http://127.0.0.1:65238/solr/dv_coll_shard3_replica_n4/&wt=javabin&version=2}{deleteByQuery=*:* (-1573905557020999680)} 0 10
   [junit4]   2> 1814737 INFO  (qtp2106332852-21937) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard1_replica_n1]  webapp=/solr path=/update params={update.distrib=TOLEADER&distrib.from=http://127.0.0.1:65238/solr/dv_coll_shard3_replica_n4/&wt=javabin&version=2}{deleteByQuery=*:* (-1573905557022048256)} 0 10
   [junit4]   2> 1814737 INFO  (qtp2106332852-21943) [n:127.0.0.1:65238_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard3_replica_n4]  webapp=/solr path=/update params={wt=javabin&version=2}{deleteByQuery=*:* (-1573905557019951104)} 0 13
   [junit4]   2> 1814739 INFO  (qtp2108814188-21933) [n:127.0.0.1:39999_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n5] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1573905557035679744,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 1814739 INFO  (qtp2108814188-21933) [n:127.0.0.1:39999_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n5] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 1814747 INFO  (qtp2108814188-21933) [n:127.0.0.1:39999_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n5] o.a.s.s.SolrIndexSearcher Opening [Searcher@42632e03[dv_coll_shard4_replica_n5] main]
   [junit4]   2> 1814747 INFO  (qtp2108814188-21933) [n:127.0.0.1:39999_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n5] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 1814748 INFO  (searcherExecutor-6871-thread-1-processing-n:127.0.0.1:39999_solr x:dv_coll_shard4_replica_n5 s:shard4 c:dv_coll r:core_node8) [n:127.0.0.1:39999_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n5] o.a.s.c.SolrCore [dv_coll_shard4_replica_n5] Registered new searcher Searcher@42632e03[dv_coll_shard4_replica_n5] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1814748 INFO  (qtp2108814188-21933) [n:127.0.0.1:39999_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n5] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard4_replica_n5]  webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:65238/solr/dv_coll_shard3_replica_n4/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 9
   [junit4]   2> 1814750 INFO  (qtp2108814188-21931) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1573905557047214080,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 1814750 INFO  (qtp2108814188-21931) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 1814753 INFO  (qtp2108814188-21931) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 1814753 INFO  (qtp2108814188-21931) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard2_replica_n2]  webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:65238/solr/dv_coll_shard3_replica_n4/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 3
   [junit4]   2> 1814760 INFO  (qtp2106332852-21936) [n:127.0.0.1:65238_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1573905557057699840,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 1814760 INFO  (qtp2106332852-21944) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1573905557057699840,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 1814760 INFO  (qtp2106332852-21944) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 1814760 INFO  (qtp2106332852-21936) [n:127.0.0.1:65238_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 1814815 INFO  (Thread-5184) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1814815 INFO  (Thread-5184) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1814816 INFO  (Thread-5184) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3dee2c5b[dv_coll_shard1_replica_n1] main]
   [junit4]   2> 1814820 INFO  (qtp2106332852-21936) [n:127.0.0.1:65238_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.s.SolrIndexSearcher Opening [Searcher@211520bc[dv_coll_shard3_replica_n4] main]
   [junit4]   2> 1814820 INFO  (qtp2106332852-21944) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 1814820 INFO  (qtp2106332852-21944) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard1_replica_n1]  webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:65238/solr/dv_coll_shard3_replica_n4/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 60
   [junit4]   2> 1814820 INFO  (qtp2106332852-21936) [n:127.0.0.1:65238_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 1814821 INFO  (searcherExecutor-6870-thread-1-processing-n:127.0.0.1:65238_solr x:dv_coll_shard3_replica_n4 s:shard3 c:dv_coll r:core_node7) [n:127.0.0.1:65238_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.c.SolrCore [dv_coll_shard3_replica_n4] Registered new searcher Searcher@211520bc[dv_coll_shard3_replica_n4] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1814822 INFO  (qtp2106332852-21936) [n:127.0.0.1:65238_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard3_replica_n4]  webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:65238/solr/dv_coll_shard3_replica_n4/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 62
   [junit4]   2> 1814822 INFO  (qtp2106332852-21938) [n:127.0.0.1:65238_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard3_replica_n4]  webapp=/solr path=/update params={_stateVer_=dv_coll:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 84
   [junit4]   2> 1814822 INFO  (Thread-5184) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1814826 INFO  (qtp2106332852-21947) [n:127.0.0.1:65238_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard3_replica_n4]  webapp=/solr path=/update params={_stateVer_=dv_coll:4&wt=javabin&version=2}{add=[2 (1573905557125857280), 3 (1573905557126905856)]} 0 2
   [junit4]   2> 1814830 INFO  (qtp2108814188-21926) [n:127.0.0.1:39999_solr c:dv_coll s:shard2 r:core_node6 x:dv_coll_shard2_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard2_replica_n2]  webapp=/solr path=/update params={_stateVer_=dv_coll:4&wt=javabin&version=2}{add=[4 (1573905557131100160)]} 0 0
   [junit4]   2> 1814833 INFO  (qtp2106332852-21940) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard1_replica_n1]  webapp=/solr path=/update params={_stateVer_=dv_coll:4&wt=javabin&version=2}{add=[1 (1573905557129003008)]} 0 4
   [junit4]   2> 1814834 INFO  (qtp2106332852-21937) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1573905557135294464,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 1814834 INFO  (qtp2106332852-21937) [n:127.0.0.1:65238_solr c:dv_coll s:shard1 r:core_nod

[...truncated too long message...]

-n:127.0.0.1:55746_solr) [n:127.0.0.1:55746_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
   [junit4]   2> 184747 INFO  (zkCallback-101-thread-2-processing-n:127.0.0.1:36257_solr) [n:127.0.0.1:36257_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
   [junit4]   2> 184768 INFO  (coreCloseExecutor-699-thread-1) [n:127.0.0.1:36257_solr c:mainCorpus s:shard1 r:core_node2 x:mainCorpus_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.mainCorpus.shard1.leader, tag=1276853719
   [junit4]   2> 184773 INFO  (jetty-closer-83-thread-4) [    ] o.a.s.c.Overseer Overseer (id=98369171605422088-127.0.0.1:36257_solr-n_0000000000) closing
   [junit4]   2> 184773 INFO  (OverseerStateUpdate-98369171605422088-127.0.0.1:36257_solr-n_0000000000) [n:127.0.0.1:36257_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:36257_solr
   [junit4]   2> 184781 INFO  (coreCloseExecutor-697-thread-2) [n:127.0.0.1:60473_solr c:mainCorpus s:shard2 r:core_node4 x:mainCorpus_shard2_replica_n3] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.mainCorpus.shard2.leader, tag=1171452201
   [junit4]   2> 184785 INFO  (jetty-closer-83-thread-4) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@43387fa2{/solr,null,UNAVAILABLE}
   [junit4]   2> 184787 INFO  (jetty-closer-83-thread-3) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@221cf663{/solr,null,UNAVAILABLE}
   [junit4]   2> 184787 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15d7a3d8a64000c, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 184788 INFO  (zkCallback-103-thread-1-processing-n:127.0.0.1:55746_solr) [n:127.0.0.1:55746_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:55746_solr
   [junit4]   2> 184788 INFO  (zkCallback-103-thread-2-processing-n:127.0.0.1:55746_solr) [n:127.0.0.1:55746_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (1)
   [junit4]   2> 184795 INFO  (coreCloseExecutor-698-thread-3) [n:127.0.0.1:55746_solr c:destination s:shard2 r:core_node4 x:destination_shard2_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.destination.shard2.leader, tag=1590337067
   [junit4]   2> 184795 INFO  (coreCloseExecutor-698-thread-2) [n:127.0.0.1:55746_solr c:workQueue s:shard2 r:core_node4 x:workQueue_shard2_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.workQueue.shard2.replica_n2, tag=1244104127
   [junit4]   2> 184810 INFO  (coreCloseExecutor-698-thread-2) [n:127.0.0.1:55746_solr c:workQueue s:shard2 r:core_node4 x:workQueue_shard2_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.workQueue.shard2.leader, tag=1244104127
   [junit4]   2> 186315 WARN  (zkCallback-103-thread-1-processing-n:127.0.0.1:55746_solr) [n:127.0.0.1:55746_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 186316 INFO  (jetty-closer-83-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@7a1b2e49{/solr,null,UNAVAILABLE}
   [junit4]   2> 186319 ERROR (SUITE-StreamExpressionTest-seed#[4381740640705C3C]-worker) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 186320 INFO  (SUITE-StreamExpressionTest-seed#[4381740640705C3C]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:50991 50991
   [junit4]   2> 186838 INFO  (Thread-67) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:50991 50991
   [junit4]   2> 186839 WARN  (Thread-67) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	88	/solr/aliases.json
   [junit4]   2> 	11	/solr/configs/conf
   [junit4]   2> 	8	/solr/configs/ml
   [junit4]   2> 	4	/solr/security.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	88	/solr/clusterprops.json
   [junit4]   2> 	88	/solr/clusterstate.json
   [junit4]   2> 	40	/solr/collections/destinationCollection/state.json
   [junit4]   2> 	26	/solr/collections/parallelDestinationCollection1/state.json
   [junit4]   2> 	20	/solr/collections/destination/state.json
   [junit4]   2> 	20	/solr/collections/parallelDestinationCollection/state.json
   [junit4]   2> 	20	/solr/collections/mainCorpus/state.json
   [junit4]   2> 	18	/solr/collections/workQueue/state.json
   [junit4]   2> 	10	/solr/collections/collection1/state.json
   [junit4]   2> 	10	/solr/collections/uknownCollection/state.json
   [junit4]   2> 	10	/solr/collections/modelCollection/state.json
   [junit4]   2> 	10	/solr/collections/checkpointCollection/state.json
   [junit4]   2> 	2	/solr/overseer_elect/election/98369171605422091-127.0.0.1:55746_solr-n_0000000001
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	88	/solr/live_nodes
   [junit4]   2> 	88	/solr/collections
   [junit4]   2> 
   [junit4]   2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.io.stream.StreamExpressionTest_4381740640705C3C-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {name_s=PostingsFormat(name=Memory), expr_s=PostingsFormat(name=Direct), terms_ss=PostingsFormat(name=MockRandom), join2_s=PostingsFormat(name=MockRandom), field_s=PostingsFormat(name=LuceneVarGapFixedInterval), multiDefault=PostingsFormat(name=MockRandom), subject=PostingsFormat(name=MockRandom), a1_s=PostingsFormat(name=MockRandom), a2_s=PostingsFormat(name=Memory), level1_s=PostingsFormat(name=Direct), body_t=PostingsFormat(name=Memory), s_multi=PostingsFormat(name=Memory), col_s=PostingsFormat(name=MockRandom), level2_s=PostingsFormat(name=LuceneVarGapFixedInterval), a_ss=PostingsFormat(name=LuceneVarGapFixedInterval), a_s=PostingsFormat(name=Memory), tv_text=PostingsFormat(name=LuceneVarGapFixedInterval), term_s=PostingsFormat(name=MockRandom), a_t=PostingsFormat(name=LuceneVarGapFixedInterval), text_s=PostingsFormat(name=Direct), id=PostingsFormat(name=Direct), text=PostingsFormat(name=Memory), featureSet_s=PostingsFormat(name=MockRandom), checkpoint_ss=PostingsFormat(name=Memory), test_t=PostingsFormat(name=Memory), ident_s=PostingsFormat(name=MockRandom), whitetok=PostingsFormat(name=Memory), side_s=PostingsFormat(name=Memory)}, docValues:{name_s=DocValuesFormat(name=Asserting), expr_s=DocValuesFormat(name=Lucene70), iteration_i=DocValuesFormat(name=Lucene70), terms_ss=DocValuesFormat(name=Lucene70), join2_s=DocValuesFormat(name=Lucene70), field_s=DocValuesFormat(name=Memory), multiDefault=DocValuesFormat(name=Lucene70), price_f=DocValuesFormat(name=Lucene70), a1_s=DocValuesFormat(name=Lucene70), a_f=DocValuesFormat(name=Lucene70), intDefault=DocValuesFormat(name=Memory), join1_i=DocValuesFormat(name=Lucene70), falseNegative_i=DocValuesFormat(name=Memory), a2_s=DocValuesFormat(name=Asserting), a_i=DocValuesFormat(name=Lucene70), join3_i=DocValuesFormat(name=Asserting), test_i=DocValuesFormat(name=Lucene70), falsePositive_i=DocValuesFormat(name=Memory), field_i=DocValuesFormat(name=Lucene70), level1_s=DocValuesFormat(name=Lucene70), s_multi=DocValuesFormat(name=Asserting), col_s=DocValuesFormat(name=Lucene70), level2_s=DocValuesFormat(name=Memory), a_ss=DocValuesFormat(name=Memory), a_s=DocValuesFormat(name=Asserting), score_f=DocValuesFormat(name=Asserting), term_s=DocValuesFormat(name=Lucene70), text_s=DocValuesFormat(name=Lucene70), id=DocValuesFormat(name=Lucene70), timestamp=DocValuesFormat(name=Memory), i_multi=DocValuesFormat(name=Lucene70), featureSet_s=DocValuesFormat(name=Lucene70), trueNegative_i=DocValuesFormat(name=Asserting), index_i=DocValuesFormat(name=Lucene70), b_ls=DocValuesFormat(name=Lucene70), test_dt=DocValuesFormat(name=Lucene70), out_i=DocValuesFormat(name=Lucene70), checkpoint_ss=DocValuesFormat(name=Asserting), ident_s=DocValuesFormat(name=Lucene70), order_i=DocValuesFormat(name=Lucene70), _version_=DocValuesFormat(name=Memory), side_s=DocValuesFormat(name=Asserting), truePositive_i=DocValuesFormat(name=Lucene70), miles_i=DocValuesFormat(name=Lucene70)}, maxPointsInLeafNode=1168, maxMBSortInHeap=7.281544111608713, sim=RandomSimilarity(queryNorm=true): {}, locale=en-ZA, timezone=US/Aleutian
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_131 (64-bit)/cpus=3,threads=1,free=145181560,total=333447168
   [junit4]   2> NOTE: All tests run in this JVM: [GreaterThanEqualToEvaluatorTest, HyperbolicCosineEvaluatorTest, TestValidatingJsonMap, TestUpdateRequestCodec, TestZkConfigManager, LessThanEqualToEvaluatorTest, JdbcDriverTest, TestClusteringResponse, BasicHttpSolrClientTest, AbsoluteValueEvaluatorTest, TestBatchUpdate, TestCoreAdmin, TestConfigSetAdminRequest, JsonValidatorTest, TestXMLEscaping, TermsResponseTest, SolrPingTest, CoalesceEvaluatorTest, TestSuggesterResponse, OrEvaluatorTest, CloudSolrClientTest, RawValueEvaluatorTest, CubedRootEvaluatorTest, CloudSolrClientCacheTest, FieldAnalysisResponseTest, HyperbolicTangentEvaluatorTest, FieldEvaluatorTest, StreamExpressionTest]
   [junit4] Completed [116/135 (1!)] on J0 in 123.50s, 91 tests, 1 error <<< FAILURES!

[...truncated 42554 lines...]

[JENKINS] Lucene-Solr-7.x-Solaris (64bit/jdk1.8.0) - Build # 73 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Solaris/73/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseSerialGC

3 tests failed.
FAILED:  org.apache.solr.cloud.TestPullReplica.testKillLeader

Error Message:
Replica state not updated in cluster state null Live Nodes: [127.0.0.1:41809_solr, 127.0.0.1:36496_solr] Last available state: DocCollection(pull_replica_test_kill_leader//collections/pull_replica_test_kill_leader/state.json/6)={   "pullReplicas":"1",   "replicationFactor":"1",   "shards":{"shard1":{       "range":"80000000-7fffffff",       "state":"active",       "replicas":{         "core_node3":{           "core":"pull_replica_test_kill_leader_shard1_replica_n1",           "base_url":"http://127.0.0.1:36496/solr",           "node_name":"127.0.0.1:36496_solr",           "state":"down",           "type":"NRT",           "leader":"true"},         "core_node4":{           "core":"pull_replica_test_kill_leader_shard1_replica_p2",           "base_url":"http://127.0.0.1:41809/solr",           "node_name":"127.0.0.1:41809_solr",           "state":"active",           "type":"PULL"}}}},   "router":{"name":"compositeId"},   "maxShardsPerNode":"100",   "autoAddReplicas":"false",   "nrtReplicas":"1",   "tlogReplicas":"0"}

Stack Trace:
java.lang.AssertionError: Replica state not updated in cluster state
null
Live Nodes: [127.0.0.1:41809_solr, 127.0.0.1:36496_solr]
Last available state: DocCollection(pull_replica_test_kill_leader//collections/pull_replica_test_kill_leader/state.json/6)={
  "pullReplicas":"1",
  "replicationFactor":"1",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node3":{
          "core":"pull_replica_test_kill_leader_shard1_replica_n1",
          "base_url":"http://127.0.0.1:36496/solr",
          "node_name":"127.0.0.1:36496_solr",
          "state":"down",
          "type":"NRT",
          "leader":"true"},
        "core_node4":{
          "core":"pull_replica_test_kill_leader_shard1_replica_p2",
          "base_url":"http://127.0.0.1:41809/solr",
          "node_name":"127.0.0.1:41809_solr",
          "state":"active",
          "type":"PULL"}}}},
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"100",
  "autoAddReplicas":"false",
  "nrtReplicas":"1",
  "tlogReplicas":"0"}
	at __randomizedtesting.SeedInfo.seed([47ADDA8921218EB3:EBB2E3D439A1AE5]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:269)
	at org.apache.solr.cloud.TestPullReplica.doTestNoLeader(TestPullReplica.java:401)
	at org.apache.solr.cloud.TestPullReplica.testKillLeader(TestPullReplica.java:290)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation

Error Message:
3 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation:     1) Thread[id=32006, name=jetty-launcher-4774-thread-2-SendThread(127.0.0.1:35430), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)    2) Thread[id=32007, name=jetty-launcher-4774-thread-2-EventThread, state=WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)    3) Thread[id=31959, name=jetty-launcher-4774-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 3 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation: 
   1) Thread[id=32006, name=jetty-launcher-4774-thread-2-SendThread(127.0.0.1:35430), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
   2) Thread[id=32007, name=jetty-launcher-4774-thread-2-EventThread, state=WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
   3) Thread[id=31959, name=jetty-launcher-4774-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
        at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
        at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
        at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
        at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
        at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
        at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
        at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
        at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
	at __randomizedtesting.SeedInfo.seed([47ADDA8921218EB3]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=32006, name=jetty-launcher-4774-thread-2-SendThread(127.0.0.1:35430), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=32006, name=jetty-launcher-4774-thread-2-SendThread(127.0.0.1:35430), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
	at __randomizedtesting.SeedInfo.seed([47ADDA8921218EB3]:0)




Build Log:
[...truncated 11121 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestPullReplica
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_47ADDA8921218EB3-001/init-core-data-001
   [junit4]   2> 510173 WARN  (SUITE-TestPullReplica-seed#[47ADDA8921218EB3]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=44 numCloses=44
   [junit4]   2> 510173 INFO  (SUITE-TestPullReplica-seed#[47ADDA8921218EB3]-worker) [    ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 510174 INFO  (SUITE-TestPullReplica-seed#[47ADDA8921218EB3]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 510174 INFO  (SUITE-TestPullReplica-seed#[47ADDA8921218EB3]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 510175 INFO  (SUITE-TestPullReplica-seed#[47ADDA8921218EB3]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_47ADDA8921218EB3-001/tempDir-001
   [junit4]   2> 510175 INFO  (SUITE-TestPullReplica-seed#[47ADDA8921218EB3]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 510175 INFO  (Thread-3371) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 510175 INFO  (Thread-3371) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 510177 ERROR (Thread-3371) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 510275 INFO  (SUITE-TestPullReplica-seed#[47ADDA8921218EB3]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:59178
   [junit4]   2> 510282 INFO  (jetty-launcher-988-thread-2) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 510282 INFO  (jetty-launcher-988-thread-1) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 510283 INFO  (jetty-launcher-988-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3ae1b81{/solr,null,AVAILABLE}
   [junit4]   2> 510284 INFO  (jetty-launcher-988-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@180b8a68{/solr,null,AVAILABLE}
   [junit4]   2> 510284 INFO  (jetty-launcher-988-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@736a2360{HTTP/1.1,[http/1.1]}{127.0.0.1:41809}
   [junit4]   2> 510284 INFO  (jetty-launcher-988-thread-2) [    ] o.e.j.s.Server Started @512661ms
   [junit4]   2> 510284 INFO  (jetty-launcher-988-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=41809}
   [junit4]   2> 510284 INFO  (jetty-launcher-988-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@557130ac{HTTP/1.1,[http/1.1]}{127.0.0.1:36496}
   [junit4]   2> 510284 INFO  (jetty-launcher-988-thread-1) [    ] o.e.j.s.Server Started @512661ms
   [junit4]   2> 510284 ERROR (jetty-launcher-988-thread-2) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 510284 INFO  (jetty-launcher-988-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=36496}
   [junit4]   2> 510284 INFO  (jetty-launcher-988-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.1.0
   [junit4]   2> 510284 INFO  (jetty-launcher-988-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 510284 INFO  (jetty-launcher-988-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 510284 ERROR (jetty-launcher-988-thread-1) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 510284 INFO  (jetty-launcher-988-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-07-25T23:51:25.696Z
   [junit4]   2> 510284 INFO  (jetty-launcher-988-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.1.0
   [junit4]   2> 510284 INFO  (jetty-launcher-988-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 510285 INFO  (jetty-launcher-988-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 510285 INFO  (jetty-launcher-988-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-07-25T23:51:25.697Z
   [junit4]   2> 510289 INFO  (jetty-launcher-988-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 510289 INFO  (jetty-launcher-988-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 510296 INFO  (jetty-launcher-988-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:59178/solr
   [junit4]   2> 510296 INFO  (jetty-launcher-988-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:59178/solr
   [junit4]   2> 510300 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15d7c2992940003, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 510301 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15d7c2992940004, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 510435 INFO  (jetty-launcher-988-thread-2) [n:127.0.0.1:41809_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 510435 INFO  (jetty-launcher-988-thread-1) [n:127.0.0.1:36496_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 510435 INFO  (jetty-launcher-988-thread-1) [n:127.0.0.1:36496_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:36496_solr
   [junit4]   2> 510436 INFO  (jetty-launcher-988-thread-2) [n:127.0.0.1:41809_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41809_solr
   [junit4]   2> 510436 INFO  (jetty-launcher-988-thread-1) [n:127.0.0.1:36496_solr    ] o.a.s.c.Overseer Overseer (id=98371284866695173-127.0.0.1:36496_solr-n_0000000000) starting
   [junit4]   2> 510438 INFO  (zkCallback-1000-thread-1-processing-n:127.0.0.1:36496_solr) [n:127.0.0.1:36496_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 510438 INFO  (zkCallback-999-thread-1-processing-n:127.0.0.1:41809_solr) [n:127.0.0.1:41809_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 510445 INFO  (jetty-launcher-988-thread-1) [n:127.0.0.1:36496_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:36496_solr
   [junit4]   2> 510447 INFO  (zkCallback-999-thread-1-processing-n:127.0.0.1:41809_solr) [n:127.0.0.1:41809_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 510447 INFO  (zkCallback-1000-thread-1-processing-n:127.0.0.1:36496_solr) [n:127.0.0.1:36496_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 510521 INFO  (jetty-launcher-988-thread-1) [n:127.0.0.1:36496_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_36496.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7634f643
   [junit4]   2> 510523 INFO  (jetty-launcher-988-thread-1) [n:127.0.0.1:36496_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_36496.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7634f643
   [junit4]   2> 510524 INFO  (jetty-launcher-988-thread-1) [n:127.0.0.1:36496_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_36496.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7634f643
   [junit4]   2> 510524 INFO  (jetty-launcher-988-thread-1) [n:127.0.0.1:36496_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_47ADDA8921218EB3-001/tempDir-001/node1/.
   [junit4]   2> 510553 INFO  (jetty-launcher-988-thread-2) [n:127.0.0.1:41809_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41809.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7634f643
   [junit4]   2> 510555 INFO  (jetty-launcher-988-thread-2) [n:127.0.0.1:41809_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41809.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7634f643
   [junit4]   2> 510555 INFO  (jetty-launcher-988-thread-2) [n:127.0.0.1:41809_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41809.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7634f643
   [junit4]   2> 510556 INFO  (jetty-launcher-988-thread-2) [n:127.0.0.1:41809_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_47ADDA8921218EB3-001/tempDir-001/node2/.
   [junit4]   2> 510575 INFO  (SUITE-TestPullReplica-seed#[47ADDA8921218EB3]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 510576 INFO  (SUITE-TestPullReplica-seed#[47ADDA8921218EB3]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:59178/solr ready
   [junit4]   2> 510580 INFO  (SUITE-TestPullReplica-seed#[47ADDA8921218EB3]-worker) [    ] o.a.s.c.TestPullReplica Using legacyCloud?: false
   [junit4]   2> 510582 INFO  (qtp58711748-7890) [n:127.0.0.1:41809_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :clusterprop with params val=false&name=legacyCloud&action=CLUSTERPROP&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 510583 INFO  (qtp58711748-7890) [n:127.0.0.1:41809_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={val=false&name=legacyCloud&action=CLUSTERPROP&wt=javabin&version=2} status=0 QTime=1
   [junit4]   2> 510601 INFO  (TEST-TestPullReplica.testAddDocs-seed#[47ADDA8921218EB3]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testAddDocs
   [junit4]   2> 510602 INFO  (qtp58711748-7893) [n:127.0.0.1:41809_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params pullReplicas=2&replicationFactor=1&collection.configName=conf&maxShardsPerNode=100&name=pull_replica_test_add_docs&nrtReplicas=1&action=CREATE&numShards=1&tlogReplicas=0&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 510604 INFO  (OverseerThreadFactory-2671-thread-1-processing-n:127.0.0.1:36496_solr) [n:127.0.0.1:36496_solr    ] o.a.s.c.CreateCollectionCmd Create collection pull_replica_test_add_docs
   [junit4]   2> 510605 WARN  (OverseerThreadFactory-2671-thread-1-processing-n:127.0.0.1:36496_solr) [n:127.0.0.1:36496_solr    ] o.a.s.c.CreateCollectionCmd Specified number of replicas of 3 on collection pull_replica_test_add_docs is higher than the number of Solr instances currently live or live and part of your createNodeSet(2). It's unusual to run two replica of the same slice on the same Solr-instance.
   [junit4]   2> 510719 INFO  (OverseerStateUpdate-98371284866695173-127.0.0.1:36496_solr-n_0000000000) [n:127.0.0.1:36496_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"pull_replica_test_add_docs",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"pull_replica_test_add_docs_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:36496/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 510725 INFO  (OverseerStateUpdate-98371284866695173-127.0.0.1:36496_solr-n_0000000000) [n:127.0.0.1:36496_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"pull_replica_test_add_docs",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"pull_replica_test_add_docs_shard1_replica_p2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:41809/solr",
   [junit4]   2>   "type":"PULL"} 
   [junit4]   2> 510733 INFO  (OverseerStateUpdate-98371284866695173-127.0.0.1:36496_solr-n_0000000000) [n:127.0.0.1:36496_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"pull_replica_test_add_docs",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"pull_replica_test_add_docs_shard1_replica_p4",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:36496/solr",
   [junit4]   2>   "type":"PULL"} 
   [junit4]   2> 510939 INFO  (qtp1216397067-7904) [n:127.0.0.1:36496_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=pull_replica_test_add_docs_shard1_replica_n1&action=CREATE&numShards=1&collection=pull_replica_test_add_docs&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 510940 INFO  (qtp1216397067-7904) [n:127.0.0.1:36496_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 510941 INFO  (qtp58711748-7891) [n:127.0.0.1:41809_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node5&collection.configName=conf&newCollection=true&name=pull_replica_test_add_docs_shard1_replica_p2&action=CREATE&numShards=1&collection=pull_replica_test_add_docs&shard=shard1&wt=javabin&version=2&replicaType=PULL
   [junit4]   2> 510941 INFO  (qtp1216397067-7906) [n:127.0.0.1:36496_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node6&collection.configName=conf&newCollection=true&name=pull_replica_test_add_docs_shard1_replica_p4&action=CREATE&numShards=1&collection=pull_replica_test_add_docs&shard=shard1&wt=javabin&version=2&replicaType=PULL
   [junit4]   2> 510941 INFO  (qtp58711748-7891) [n:127.0.0.1:41809_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 511051 INFO  (zkCallback-1000-thread-1-processing-n:127.0.0.1:36496_solr) [n:127.0.0.1:36496_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_add_docs/state.json] for collection [pull_replica_test_add_docs] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 511051 INFO  (zkCallback-999-thread-1-processing-n:127.0.0.1:41809_solr) [n:127.0.0.1:41809_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_add_docs/state.json] for collection [pull_replica_test_add_docs] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 511052 INFO  (zkCallback-1000-thread-2-processing-n:127.0.0.1:36496_solr) [n:127.0.0.1:36496_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_add_docs/state.json] for collection [pull_replica_test_add_docs] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 511052 INFO  (zkCallback-999-thread-2-processing-n:127.0.0.1:41809_solr) [n:127.0.0.1:41809_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_add_docs/state.json] for collection [pull_replica_test_add_docs] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 511954 INFO  (qtp1216397067-7906) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node6 x:pull_replica_test_add_docs_shard1_replica_p4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 511954 INFO  (qtp1216397067-7904) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 511955 INFO  (qtp58711748-7891) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 511969 INFO  (qtp58711748-7891) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.s.IndexSchema [pull_replica_test_add_docs_shard1_replica_p2] Schema name=minimal
   [junit4]   2> 511969 INFO  (qtp1216397067-7906) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node6 x:pull_replica_test_add_docs_shard1_replica_p4] o.a.s.s.IndexSchema [pull_replica_test_add_docs_shard1_replica_p4] Schema name=minimal
   [junit4]   2> 511971 INFO  (qtp58711748-7891) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 511971 INFO  (qtp58711748-7891) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.c.CoreContainer Creating SolrCore 'pull_replica_test_add_docs_shard1_replica_p2' using configuration from collection pull_replica_test_add_docs, trusted=true
   [junit4]   2> 511971 INFO  (qtp1216397067-7906) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node6 x:pull_replica_test_add_docs_shard1_replica_p4] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 511971 INFO  (qtp1216397067-7906) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node6 x:pull_replica_test_add_docs_shard1_replica_p4] o.a.s.c.CoreContainer Creating SolrCore 'pull_replica_test_add_docs_shard1_replica_p4' using configuration from collection pull_replica_test_add_docs, trusted=true
   [junit4]   2> 511971 INFO  (qtp58711748-7891) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41809.solr.core.pull_replica_test_add_docs.shard1.replica_p2' (registry 'solr.core.pull_replica_test_add_docs.shard1.replica_p2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7634f643
   [junit4]   2> 511971 INFO  (qtp1216397067-7906) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node6 x:pull_replica_test_add_docs_shard1_replica_p4] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_36496.solr.core.pull_replica_test_add_docs.shard1.replica_p4' (registry 'solr.core.pull_replica_test_add_docs.shard1.replica_p4') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7634f643
   [junit4]   2> 511971 INFO  (qtp58711748-7891) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 511971 INFO  (qtp1216397067-7906) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node6 x:pull_replica_test_add_docs_shard1_replica_p4] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 511971 INFO  (qtp58711748-7891) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.c.SolrCore [[pull_replica_test_add_docs_shard1_replica_p2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_47ADDA8921218EB3-001/tempDir-001/node2/pull_replica_test_add_docs_shard1_replica_p2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_47ADDA8921218EB3-001/tempDir-001/node2/./pull_replica_test_add_docs_shard1_replica_p2/data/]
   [junit4]   2> 511971 INFO  (qtp1216397067-7906) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node6 x:pull_replica_test_add_docs_shard1_replica_p4] o.a.s.c.SolrCore [[pull_replica_test_add_docs_shard1_replica_p4] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_47ADDA8921218EB3-001/tempDir-001/node1/pull_replica_test_add_docs_shard1_replica_p4], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_47ADDA8921218EB3-001/tempDir-001/node1/./pull_replica_test_add_docs_shard1_replica_p4/data/]
   [junit4]   2> 511977 INFO  (qtp1216397067-7904) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.s.IndexSchema [pull_replica_test_add_docs_shard1_replica_n1] Schema name=minimal
   [junit4]   2> 511978 INFO  (qtp1216397067-7904) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 511978 INFO  (qtp1216397067-7904) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'pull_replica_test_add_docs_shard1_replica_n1' using configuration from collection pull_replica_test_add_docs, trusted=true
   [junit4]   2> 511978 INFO  (qtp1216397067-7904) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_36496.solr.core.pull_replica_test_add_docs.shard1.replica_n1' (registry 'solr.core.pull_replica_test_add_docs.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7634f643
   [junit4]   2> 511978 INFO  (qtp1216397067-7904) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 511978 INFO  (qtp1216397067-7904) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.SolrCore [[pull_replica_test_add_docs_shard1_replica_n1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_47ADDA8921218EB3-001/tempDir-001/node1/pull_replica_test_add_docs_shard1_replica_n1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_47ADDA8921218EB3-001/tempDir-001/node1/./pull_replica_test_add_docs_shard1_replica_n1/data/]
   [junit4]   2> 512029 INFO  (qtp1216397067-7906) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node6 x:pull_replica_test_add_docs_shard1_replica_p4] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 512029 INFO  (qtp1216397067-7906) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node6 x:pull_replica_test_add_docs_shard1_replica_p4] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 512031 INFO  (qtp1216397067-7906) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node6 x:pull_replica_test_add_docs_shard1_replica_p4] o.a.s.s.SolrIndexSearcher Opening [Searcher@70439828[pull_replica_test_add_docs_shard1_replica_p4] main]
   [junit4]   2> 512032 INFO  (qtp1216397067-7906) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node6 x:pull_replica_test_add_docs_shard1_replica_p4] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 512033 INFO  (qtp1216397067-7906) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node6 x:pull_replica_test_add_docs_shard1_replica_p4] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 512033 INFO  (qtp1216397067-7906) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node6 x:pull_replica_test_add_docs_shard1_replica_p4] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 512034 INFO  (searcherExecutor-2677-thread-1-processing-n:127.0.0.1:36496_solr x:pull_replica_test_add_docs_shard1_replica_p4 s:shard1 c:pull_replica_test_add_docs r:core_node6) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node6 x:pull_replica_test_add_docs_shard1_replica_p4] o.a.s.c.SolrCore [pull_replica_test_add_docs_shard1_replica_p4] Registered new searcher Searcher@70439828[pull_replica_test_add_docs_shard1_replica_p4] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 512035 INFO  (qtp1216397067-7906) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node6 x:pull_replica_test_add_docs_shard1_replica_p4] o.a.s.c.ZkController pull_replica_test_add_docs_shard1_replica_p4 starting background replication from leader
   [junit4]   2> 512035 INFO  (qtp1216397067-7906) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node6 x:pull_replica_test_add_docs_shard1_replica_p4] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03
   [junit4]   2> 512036 INFO  (qtp1216397067-7906) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node6 x:pull_replica_test_add_docs_shard1_replica_p4] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms
   [junit4]   2> 512046 INFO  (qtp58711748-7891) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 512046 INFO  (qtp58711748-7891) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 512047 INFO  (qtp58711748-7891) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.s.SolrIndexSearcher Opening [Searcher@229792f8[pull_replica_test_add_docs_shard1_replica_p2] main]
   [junit4]   2> 512049 INFO  (qtp58711748-7891) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 512049 INFO  (qtp58711748-7891) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 512050 INFO  (qtp58711748-7891) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 512051 INFO  (qtp1216397067-7904) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 512051 INFO  (qtp1216397067-7904) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 512051 INFO  (qtp58711748-7891) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.c.ZkController pull_replica_test_add_docs_shard1_replica_p2 starting background replication from leader
   [junit4]   2> 512051 INFO  (qtp58711748-7891) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03
   [junit4]   2> 512051 INFO  (searcherExecutor-2676-thread-1-processing-n:127.0.0.1:41809_solr x:pull_replica_test_add_docs_shard1_replica_p2 s:shard1 c:pull_replica_test_add_docs r:core_node5) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.c.SolrCore [pull_replica_test_add_docs_shard1_replica_p2] Registered new searcher Searcher@229792f8[pull_replica_test_add_docs_shard1_replica_p2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 512052 INFO  (qtp1216397067-7904) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 512052 INFO  (qtp1216397067-7904) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 512052 INFO  (qtp58711748-7891) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms
   [junit4]   2> 512053 INFO  (qtp1216397067-7904) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@391f3b78[pull_replica_test_add_docs_shard1_replica_n1] main]
   [junit4]   2> 512055 INFO  (qtp1216397067-7904) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 512056 INFO  (qtp1216397067-7904) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 512056 INFO  (qtp1216397067-7904) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 512057 INFO  (qtp1216397067-7904) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1573940559839494144
   [junit4]   2> 512057 INFO  (searcherExecutor-2678-thread-1-processing-n:127.0.0.1:36496_solr x:pull_replica_test_add_docs_shard1_replica_n1 s:shard1 c:pull_replica_test_add_docs r:core_node3) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.SolrCore [pull_replica_test_add_docs_shard1_replica_n1] Registered new searcher Searcher@391f3b78[pull_replica_test_add_docs_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 512067 INFO  (qtp1216397067-7904) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 512067 INFO  (qtp1216397067-7904) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 512067 INFO  (qtp1216397067-7904) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:36496/solr/pull_replica_test_add_docs_shard1_replica_n1/
   [junit4]   2> 512068 INFO  (qtp1216397067-7904) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 512068 INFO  (qtp1216397067-7904) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:36496/solr/pull_replica_test_add_docs_shard1_replica_n1/ has no replicas
   [junit4]   2> 512068 INFO  (qtp1216397067-7904) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 512077 INFO  (qtp1216397067-7904) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:36496/solr/pull_replica_test_add_docs_shard1_replica_n1/ shard1
   [junit4]   2> 512189 INFO  (zkCallback-999-thread-2-processing-n:127.0.0.1:41809_solr) [n:127.0.0.1:41809_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_add_docs/state.json] for collection [pull_replica_test_add_docs] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 512189 INFO  (zkCallback-1000-thread-1-processing-n:127.0.0.1:36496_solr) [n:127.0.0.1:36496_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_add_docs/state.json] for collection [pull_replica_test_add_docs] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 512189 INFO  (zkCallback-999-thread-1-processing-n:127.0.0.1:41809_solr) [n:127.0.0.1:41809_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_add_docs/state.json] for collection [pull_replica_test_add_docs] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 512189 INFO  (zkCallback-1000-thread-2-processing-n:127.0.0.1:36496_solr) [n:127.0.0.1:36496_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_add_docs/state.json] for collection [pull_replica_test_add_docs] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 512229 INFO  (qtp1216397067-7904) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 512231 INFO  (qtp1216397067-7904) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=pull_replica_test_add_docs_shard1_replica_n1&action=CREATE&numShards=1&collection=pull_replica_test_add_docs&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1292
   [junit4]   2> 512339 INFO  (zkCallback-1000-thread-2-processing-n:127.0.0.1:36496_solr) [n:127.0.0.1:36496_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_add_docs/state.json] for collection [pull_replica_test_add_docs] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 512339 INFO  (zkCallback-999-thread-1-processing-n:127.0.0.1:41809_solr) [n:127.0.0.1:41809_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_add_docs/state.json] for collection [pull_replica_test_add_docs] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 512339 INFO  (zkCallback-1000-thread-1-processing-n:127.0.0.1:36496_solr) [n:127.0.0.1:36496_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_add_docs/state.json] for collection [pull_replica_test_add_docs] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 512339 INFO  (zkCallback-999-thread-2-processing-n:127.0.0.1:41809_solr) [n:127.0.0.1:41809_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_add_docs/state.json] for collection [pull_replica_test_add_docs] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 513041 INFO  (qtp1216397067-7906) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node6 x:pull_replica_test_add_docs_shard1_replica_p4] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node6&collection.configName=conf&newCollection=true&name=pull_replica_test_add_docs_shard1_replica_p4&action=CREATE&numShards=1&collection=pull_replica_test_add_docs&shard=shard1&wt=javabin&version=2&replicaType=PULL} status=0 QTime=2101
   [junit4]   2> 513056 INFO  (qtp58711748-7891) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node5&collection.configName=conf&newCollection=true&name=pull_replica_test_add_docs_shard1_replica_p2&action=CREATE&numShards=1&collection=pull_replica_test_add_docs&shard=shard1&wt=javabin&version=2&replicaType=PULL} status=0 QTime=2116
   [junit4]   2> 513059 INFO  (qtp58711748-7893) [n:127.0.0.1:41809_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 513158 INFO  (zkCallback-1000-thread-2-processing-n:127.0.0.1:36496_solr) [n:127.0.0.1:36496_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_add_docs/state.json] for collection [pull_replica_test_add_docs] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 513158 INFO  (zkCallback-999-thread-2-processing-n:127.0.0.1:41809_solr) [n:127.0.0.1:41809_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_add_docs/state.json] for collection [pull_replica_test_add_docs] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 513158 INFO  (zkCallback-1000-thread-1-processing-n:127.0.0.1:36496_solr) [n:127.0.0.1:36496_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_add_docs/state.json] for collection [pull_replica_test_add_docs] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 513158 INFO  (zkCallback-999-thread-1-processing-n:127.0.0.1:41809_solr) [n:127.0.0.1:41809_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_add_docs/state.json] for collection [pull_replica_test_add_docs] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 514037 INFO  (indexFetcher-2692-thread-1) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.h.IndexFetcher Updated masterUrl to http://127.0.0.1:36496/solr/pull_replica_test_add_docs_shard1_replica_n1/
   [junit4]   2> 514039 INFO  (qtp1216397067-7903) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_n1]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
   [junit4]   2> 514039 INFO  (indexFetcher-2692-thread-1) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.h.IndexFetcher Master's generation: 1
   [junit4]   2> 514039 INFO  (indexFetcher-2692-thread-1) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.h.IndexFetcher Master's version: 0
   [junit4]   2> 514039 INFO  (indexFetcher-2692-thread-1) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.h.IndexFetcher Slave's generation: 1
   [junit4]   2> 514040 INFO  (indexFetcher-2692-thread-1) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.h.IndexFetcher Slave's version: 0
   [junit4]   2> 514068 INFO  (qtp58711748-7893) [n:127.0.0.1:41809_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={pullReplicas=2&replicationFactor=1&collection.configName=conf&maxShardsPerNode=100&name=pull_replica_test_add_docs&nrtReplicas=1&action=CREATE&numShards=1&tlogReplicas=0&wt=javabin&version=2} status=0 QTime=3467
   [junit4]   2> 514079 INFO  (qtp1216397067-7899) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [pull_replica_test_add_docs_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[1 (1573940561956569088)]} 0 3
   [junit4]   2> 514080 INFO  (qtp1216397067-7901) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1573940561960763392,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 514080 INFO  (qtp1216397067-7901) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@308621c2 commitCommandVersion:1573940561960763392
   [junit4]   2> 514085 INFO  (qtp1216397067-7901) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@4cc5852e[pull_replica_test_add_docs_shard1_replica_n1] main]
   [junit4]   2> 514086 INFO  (qtp1216397067-7901) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 514087 INFO  (searcherExecutor-2678-thread-1-processing-n:127.0.0.1:36496_solr x:pull_replica_test_add_docs_shard1_replica_n1 s:shard1 c:pull_replica_test_add_docs r:core_node3) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.SolrCore [pull_replica_test_add_docs_shard1_replica_n1] Registered new searcher Searcher@4cc5852e[pull_replica_test_add_docs_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.1.0):C1)))}
   [junit4]   2> 514087 INFO  (qtp1216397067-7901) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [pull_replica_test_add_docs_shard1_replica_n1]  webapp=/solr path=/update params={_stateVer_=pull_replica_test_add_docs:5&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 7
   [junit4]   2> 514091 INFO  (qtp1216397067-7906) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_n1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=1 status=0 QTime=0
   [junit4]   2> 514096 INFO  (qtp58711748-7894) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_p2]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 514197 INFO  (qtp58711748-7895) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_p2]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 514299 INFO  (qtp58711748-7890) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_p2]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 514409 INFO  (indexFetcher-2685-thread-1) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node6 x:pull_replica_test_add_docs_shard1_replica_p4] o.a.s.h.IndexFetcher Updated masterUrl to http://127.0.0.1:36496/solr/pull_replica_test_add_docs_shard1_replica_n1/
   [junit4]   2> 514409 INFO  (qtp58711748-7896) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_p2]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 514410 INFO  (qtp1216397067-7905) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_n1]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
   [junit4]   2> 514411 INFO  (indexFetcher-2685-thread-1) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node6 x:pull_replica_test_add_docs_shard1_replica_p4] o.a.s.h.IndexFetcher Master's generation: 2
   [junit4]   2> 514411 INFO  (indexFetcher-2685-thread-1) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node6 x:pull_replica_test_add_docs_shard1_replica_p4] o.a.s.h.IndexFetcher Master's version: 1501026689492
   [junit4]   2> 514411 INFO  (indexFetcher-2685-thread-1) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node6 x:pull_replica_test_add_docs_shard1_replica_p4] o.a.s.h.IndexFetcher Slave's generation: 1
   [junit4]   2> 514411 INFO  (indexFetcher-2685-thread-1) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node6 x:pull_replica_test_add_docs_shard1_replica_p4] o.a.s.h.IndexFetcher Slave's version: 0
   [junit4]   2> 514411 INFO  (indexFetcher-2685-thread-1) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node6 x:pull_replica_test_add_docs_shard1_replica_p4] o.a.s.h.IndexFetcher Starting replication process
   [junit4]   2> 514412 INFO  (qtp1216397067-7899) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_n1]  webapp=/solr path=/replication params={generation=2&qt=/replication&wt=javabin&version=2&command=filelist} status=0 QTime=0
   [junit4]   2> 514412 INFO  (indexFetcher-2685-thread-1) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node6 x:pull_replica_test_add_docs_shard1_replica_p4] o.a.s.h.IndexFetcher Number of files in latest index in master: 17
   [junit4]   2> 514416 INFO  (indexFetcher-2685-thread-1) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node6 x:pull_replica_test_add_docs_shard1_replica_p4] o.a.s.h.IndexFetcher Starting download (fullCopy=false) to MockDirectoryWrapper(RAMDirectory@65273550 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@290c1521)
   [junit4]   2> 514416 INFO  (qtp1216397067-7901) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_n1]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.si&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 514418 INFO  (qtp1216397067-7904) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_n1]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_LuceneFixedGap_0.tib&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 514420 INFO  (qtp1216397067-7906) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_n1]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_Direct_0.dvdd&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 514448 INFO  (qtp1216397067-7903) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_n1]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_LuceneVarGapFixedInterval_0.tiv&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 514449 INFO  (qtp1216397067-7900) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_n1]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.nvd&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 514450 INFO  (qtp1216397067-7905) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_n1]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_LuceneVarGapFixedInterval_0.pos&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 514451 INFO  (qtp1216397067-7899) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_n1]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.fdx&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 514452 INFO  (qtp1216397067-7901) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_n1]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_LuceneFixedGap_0.doc&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 514453 INFO  (qtp1216397067-7904) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_n1]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_LuceneVarGapFixedInterval_0.doc&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 514454 INFO  (qtp1216397067-7906) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_n1]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.fdt&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 514455 INFO  (qtp1216397067-7903) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_n1]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_LuceneFixedGap_0.pos&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 514456 INFO  (qtp1216397067-7900) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_n1]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_LuceneFixedGap_0.tii&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 514456 INFO  (qtp1216397067-7905) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_n1]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_LuceneVarGapFixedInterval_0.tib&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 514457 INFO  (qtp1216397067-7899) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_n1]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.nvm&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 514457 INFO  (qtp1216397067-7901) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_n1]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_Direct_0.dvdm&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 514458 INFO  (qtp1216397067-7904) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_n1]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.fnm&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 514459 INFO  (qtp1216397067-7906) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_n1]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=segments_2&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 514459 INFO  (indexFetcher-2685-thread-1) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node6 x:pull_replica_test_add_docs_shard1_replica_p4] o.a.s.h.IndexFetcher Total time taken for download (fullCopy=false,bytesDownloaded=2935) : 0 secs (null bytes/sec) to MockDirectoryWrapper(RAMDirectory@65273550 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@290c1521)
   [junit4]   2> 514461 INFO  (indexFetcher-2685-thread-1) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node6 x:pull_replica_test_add_docs_shard1_replica_p4] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
   [junit4]   2> 514462 INFO  (indexFetcher-2685-thread-1) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node6 x:pull_replica_test_add_docs_shard1_replica_p4] o.a.s.s.SolrIndexSearcher Opening [Searcher@5673c09[pull_replica_test_add_docs_shard1_replica_p4] main]
   [junit4]   2> 514463 INFO  (searcherExecutor-2677-thread-1-processing-n:127.0.0.1:36496_solr x:pull_replica_test_add_docs_shard1_replica_p4 s:shard1 c:pull_replica_test_add_docs r:core_node6) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node6 x:pull_replica_test_add_docs_shard1_replica_p4] o.a.s.c.SolrCore [pull_replica_test_add_docs_shard1_replica_p4] Registered new searcher Searcher@5673c09[pull_replica_test_add_docs_shard1_replica_p4] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.1.0):C1)))}
   [junit4]   2> 514510 INFO  (qtp58711748-7891) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_p2]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 514607 INFO  (OverseerCollectionConfigSetProcessor-98371284866695173-127.0.0.1:36496_solr-n_0000000000) [n:127.0.0.1:36496_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 514611 INFO  (qtp58711748-7893) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_p2]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 514713 INFO  (qtp58711748-7897) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_p2]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 514814 INFO  (qtp58711748-7894) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_p2]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 514914 INFO  (qtp58711748-7895) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_p2]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 515016 INFO  (qtp58711748-7890) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_p2]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 515119 INFO  (qtp58711748-7896) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_p2]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 515227 INFO  (qtp58711748-7891) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_p2]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 515337 INFO  (qtp58711748-7893) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_p2]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 515438 INFO  (qtp58711748-7897) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_p2]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 515539 INFO  (qtp58711748-7897) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_p2]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 515640 INFO  (qtp58711748-7895) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_p2]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 515741 INFO  (qtp58711748-7895) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_p2]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 515842 INFO  (qtp58711748-7896) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_p2]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 515943 INFO  (qtp58711748-7891) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_p2]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 516044 INFO  (qtp58711748-7893) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_p2]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 516146 INFO  (qtp58711748-7894) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_p2]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 516248 INFO  (qtp58711748-7894) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_p2]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 516350 INFO  (qtp58711748-7894) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_p2]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 516453 INFO  (qtp58711748-7895) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_p2]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 516558 INFO  (qtp58711748-7895) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_p2]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 516659 INFO  (qtp58711748-7891) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_p2]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 516776 INFO  (qtp58711748-7893) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_p2]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 516877 INFO  (qtp58711748-7897) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_p2]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 516978 INFO  (qtp58711748-7897) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_p2]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 517036 INFO  (qtp1216397067-7903) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_n1]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
   [junit4]   2> 517036 INFO  (indexFetcher-2692-thread-1) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.h.IndexFetcher Master's generation: 2
   [junit4]   2> 517036 INFO  (indexFetcher-2692-thread-1) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.h.IndexFetcher Master's version: 1501026689492
   [junit4]   2> 517036 INFO  (indexFetcher-2692-thread-1) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.h.IndexFetcher Slave's generation: 1
   [junit4]   2> 517036 INFO  (indexFetcher-2692-thread-1) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.h.IndexFetcher Slave's version: 0
   [junit4]   2> 517036 INFO  (indexFetcher-2692-thread-1) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.h.IndexFetcher Starting replication process
   [junit4]   2> 517037 INFO  (qtp1216397067-7903) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_n1]  webapp=/solr path=/replication params={generation=2&qt=/replication&wt=javabin&version=2&command=filelist} status=0 QTime=0
   [junit4]   2> 517038 INFO  (indexFetcher-2692-thread-1) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.h.IndexFetcher Number of files in latest index in master: 17
   [junit4]   2> 517041 INFO  (indexFetcher-2692-thread-1) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.h.IndexFetcher Starting download (fullCopy=false) to MockDirectoryWrapper(RAMDirectory@547ea38 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2c133765)
   [junit4]   2> 517050 INFO  (qtp1216397067-7903) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_n1]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.si&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 517052 INFO  (qtp1216397067-7899) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_n1]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_LuceneFixedGap_0.tib&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 517053 INFO  (qtp1216397067-7899) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_n1]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_Direct_0.dvdd&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 517054 INFO  (qtp1216397067-7899) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_n1]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_LuceneVarGapFixedInterval_0.tiv&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 517055 INFO  (qtp1216397067-7899) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_n1]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.nvd&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 517056 INFO  (qtp1216397067-7899) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_n1]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_LuceneVarGapFixedInterval_0.pos&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 517057 INFO  (qtp1216397067-7899) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_n1]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.fdx&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 517059 INFO  (qtp1216397067-7899) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_n1]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_LuceneFixedGap_0.doc&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 517060 INFO  (qtp1216397067-7899) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_n1]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_LuceneVarGapFixedInterval_0.doc&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 517061 INFO  (qtp1216397067-7899) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_n1]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.fdt&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 517062 INFO  (qtp1216397067-7901) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_n1]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_LuceneFixedGap_0.pos&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 517064 INFO  (qtp1216397067-7901) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_n1]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_LuceneFixedGap_0.tii&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 517065 INFO  (qtp1216397067-7901) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_n1]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_LuceneVarGapFixedInterval_0.tib&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 517066 INFO  (qtp1216397067-7901) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_n1]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.nvm&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 517067 INFO  (qtp1216397067-7901) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_n1]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_Direct_0.dvdm&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 517068 INFO  (qtp1216397067-7901) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_n1]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.fnm&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 517069 INFO  (qtp1216397067-7901) [n:127.0.0.1:36496_solr c:pull_replica_test_add_docs s:shard1 r:core_node3 x:pull_replica_test_add_docs_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_add_docs_shard1_replica_n1]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=segments_2&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 517070 INFO  (indexFetcher-2692-thread-1) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.h.IndexFetcher Total time taken for download (fullCopy=false,bytesDownloaded=2935) : 0 secs (null bytes/sec) to MockDirectoryWrapper(RAMDirectory@547ea38 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2c133765)
   [junit4]   2> 517079 INFO  (indexFetcher-2692-thread-1) [n:127.0.0.1:41809_solr c:pull_replica_test_add_docs s:shard1 r:core_node5 x:pull_replica_test_add_docs_shard1_replica_p2] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
   [junit4]   2> 517091

[...truncated too long message...]

ManagedSynonymFilterFactory, DocValuesMultiTest, TestClusterProperties, TestPushWriter, ShowFileRequestHandlerTest, TestExpandComponent, SpatialRPTFieldTypeTest, TestSolrFieldCacheBean, MoreLikeThisHandlerTest, TestRemoteStreaming, TriLevelCompositeIdRoutingTest, ChaosMonkeySafeLeaderWithPullReplicasTest, RestartWhileUpdatingTest, TestLMJelinekMercerSimilarityFactory, TestDelegationWithHadoopAuth, VersionInfoTest, CreateCollectionCleanupTest, TestRequestForwarding, DistributedQueryElevationComponentTest, TestQueryTypes, CollectionTooManyReplicasTest, DistributedTermsComponentTest, TestSolrCoreSnapshots, CdcrReplicationHandlerTest, OpenExchangeRatesOrgProviderTest, SolrCloudReportersTest, FacetPivotSmallTest, OverseerCollectionConfigSetProcessorTest, TestBackupRepositoryFactory, AutoCommitTest, TestDocumentBuilder, TestRTimerTree, TestLegacyTerms, DeleteShardTest, TestTrie, TestMacroExpander, ConvertedLegacyTest, HdfsRestartWhileUpdatingTest, BadComponentTest, DocumentBuilderTest, FullSolrCloudDistribCmdsTest, ChaosMonkeySafeLeaderTest, DistributedFacetPivotSmallAdvancedTest, TestJsonFacetRefinement, SOLR749Test, SuggesterTSTTest, TestSolrCloudWithHadoopAuthPlugin, OverseerRolesTest, ShardRoutingCustomTest, BlockCacheTest, BigEndianAscendingWordSerializerTest, TestPartialUpdateDeduplication, SearchHandlerTest, DistributedMLTComponentTest, ZkSolrClientTest, TestGraphMLResponseWriter, TestSimpleTrackingShardHandler, SolrCoreMetricManagerTest, TestQueryUtils, UpdateParamsTest, TestNumericTokenStream, TestDynamicFieldResource, UnloadDistributedZkTest, CollectionsAPIDistributedZkTest, SyncSliceTest, OpenCloseCoreStressTest, LeaderElectionIntegrationTest, ShardRoutingTest, BasicZkTest, RecoveryZkTest, TestRandomFaceting, TestZkChroot, ZkCLITest, TestRecovery, TermVectorComponentDistributedTest, ZkControllerTest, TestRealTimeGet, TestJoin, TestReload, HardAutoCommitTest, TestRangeQuery, SolrCoreTest, SpellCheckComponentTest, TestGroupingSearch, SolrCmdDistributorTest, QueryElevationComponentTest, TestSort, TestFiltering, BasicFunctionalityTest, TestLazyCores, DirectUpdateHandlerTest, TestBadConfig, HighlighterTest, SoftAutoCommitTest, SolrIndexSplitterTest, SimplePostToolTest, TestCoreDiscovery, AnalysisAfterCoreReloadTest, SignatureUpdateProcessorFactoryTest, TestExtendedDismaxParser, SpellCheckCollatorTest, SuggesterFSTTest, CoreAdminHandlerTest, SolrRequestParserTest, TestFoldingMultitermQuery, SuggesterTest, SpatialFilterTest, SuggesterWFSTTest, PolyFieldTest, WordBreakSolrSpellCheckerTest, SchemaVersionSpecificBehaviorTest, SolrCoreCheckLockOnStartupTest, TestAtomicUpdateErrorCases, TestWordDelimiterFilterFactory, DefaultValueUpdateProcessorTest, TestSolrDeletionPolicy1, DebugComponentTest, IndexBasedSpellCheckerTest, DisMaxRequestHandlerTest, TestWriterPerf, DirectSolrSpellCheckerTest, TestReversedWildcardFilterFactory, DocumentAnalysisRequestHandlerTest, FileBasedSpellCheckerTest, XmlUpdateRequestHandlerTest, PathHierarchyTokenizerFactoryTest, RequiredFieldsTest, FieldAnalysisRequestHandlerTest, TestSolrQueryParser, RegexBoostProcessorTest, TestJmxIntegration, ReturnFieldsTest, QueryParsingTest, TestAnalyzedSuggestions, UniqFieldsUpdateProcessorFactoryTest, BinaryUpdateRequestHandlerTest, TestPHPSerializedResponseWriter, CSVRequestHandlerTest, TestComponentsName, TestSolrIndexConfig, TestQuerySenderNoQuery, SampleTest, TestDocSet, NumericFieldsTest, TestElisionMultitermQuery, OutputWriterTest, TestLuceneMatchVersion, TestPhraseSuggestions, SpellPossibilityIteratorTest, TestCharFilters, TestCodecSupport, SynonymTokenizerTest, TestDFRSimilarityFactory, TestPerFieldSimilarity, TestLMDirichletSimilarityFactory, TestIBSimilarityFactory, TestBM25SimilarityFactory, TestFastWriter, TestFastOutputStream, PluginInfoTest, DateMathParserTest, DOMUtilTest, RAMDirectoryFactoryTest, ClusterStateTest, TestLRUCache, ZkNodePropsTest, SliceStateTest, FileUtilsTest, TestRTGBase, DistributedIntervalFacetingTest, TestHighlightDedupGrouping, TestTolerantSearch, ActionThrottleTest, AliasIntegrationTest, AsyncCallRequestStatusResponseTest, BaseCdcrDistributedZkTest, CdcrReplicationDistributedZkTest, CdcrVersionReplicationTest, ChaosMonkeyNothingIsSafeWithPullReplicasTest, CleanupOldIndexTest, CollectionsAPIAsyncDistributedZkTest, CollectionsAPISolrJTest, DeleteLastCustomShardedReplicaTest, DeleteNodeTest, DistribCursorPagingTest, DistribJoinFromCollectionTest, ForceLeaderTest, LeaderInitiatedRecoveryOnShardRestartTest, MigrateRouteKeyTest, OutOfBoxZkACLAndCredentialsProvidersTest, OverseerModifyCollectionTest, OverseerTaskQueueTest, RemoteQueryErrorTest, ReplaceNodeTest, RollingRestartTest, SharedFSAutoReplicaFailoverTest, SharedFSAutoReplicaFailoverUtilsTest, SimpleCollectionCreateDeleteTest, SolrCLIZkUtilsTest, SolrXmlInZkTest, TestAuthenticationFramework, TestCloudJSONFacetJoinDomain, TestCloudPivotFacet, TestCloudPseudoReturnFields, TestCollectionAPI, TestCollectionsAPIViaSolrCloudCluster, TestConfigSetsAPI, TestCryptoKeys, TestDeleteCollectionOnDownNodes, TestDistribDocBasedVersion, TestHdfsCloudBackupRestore, TestLeaderElectionWithEmptyReplica, TestLocalFSCloudBackupRestore, TestOnReconnectListenerSupport, TestPullReplicaErrorHandling, TestRandomFlRTGCloud, TestRandomRequestDistribution, TestRequestStatusCollectionAPI, TestSSLRandomization, TestShortCircuitedRequests, TestSolrCloudWithSecureImpersonation]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestSolrCloudWithSecureImpersonation -Dtests.seed=47ADDA8921218EB3 -Dtests.slow=true -Dtests.locale=sr -Dtests.timezone=Antarctica/Syowa -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   0.00s J1 | TestSolrCloudWithSecureImpersonation (suite) <<<
   [junit4]    > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 3 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation: 
   [junit4]    >    1) Thread[id=32006, name=jetty-launcher-4774-thread-2-SendThread(127.0.0.1:35430), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
   [junit4]    >    2) Thread[id=32007, name=jetty-launcher-4774-thread-2-EventThread, state=WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
   [junit4]    >    3) Thread[id=31959, name=jetty-launcher-4774-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]    >         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]    >         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]    >         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]    >         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([47ADDA8921218EB3]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   [junit4]    >    1) Thread[id=32006, name=jetty-launcher-4774-thread-2-SendThread(127.0.0.1:35430), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([47ADDA8921218EB3]:0)
   [junit4] Completed [546/726 (2!)] on J1 in 55.23s, 12 tests, 2 errors <<< FAILURES!

[...truncated 43939 lines...]