You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2018/12/11 16:33:41 UTC

[JENKINS] Lucene-Solr-Tests-master - Build # 3050 - Still unstable

Build: https://builds.apache.org/job/Lucene-Solr-Tests-master/3050/

1 tests failed.
FAILED:  org.apache.solr.cloud.DeleteReplicaTest.deleteLiveReplicaTest

Error Message:
expected:<0> but was:<1>

Stack Trace:
java.lang.AssertionError: expected:<0> but was:<1>
	at __randomizedtesting.SeedInfo.seed([B93C9E584072689C:145C2A535D4DC0E9]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.cloud.DeleteReplicaTest.deleteLiveReplicaTest(DeleteReplicaTest.java:125)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:935)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:971)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:985)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:944)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:830)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 12948 lines...]
   [junit4] Suite: org.apache.solr.cloud.DeleteReplicaTest
   [junit4]   2> 366623 INFO  (SUITE-DeleteReplicaTest-seed#[B93C9E584072689C]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.DeleteReplicaTest_B93C9E584072689C-001/init-core-data-001
   [junit4]   2> 366624 WARN  (SUITE-DeleteReplicaTest-seed#[B93C9E584072689C]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1090 numCloses=1090
   [junit4]   2> 366624 INFO  (SUITE-DeleteReplicaTest-seed#[B93C9E584072689C]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 366629 INFO  (SUITE-DeleteReplicaTest-seed#[B93C9E584072689C]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 366632 INFO  (TEST-DeleteReplicaTest.deleteReplicaOnIndexing-seed#[B93C9E584072689C]) [    ] o.a.s.SolrTestCaseJ4 ###Starting deleteReplicaOnIndexing
   [junit4]   2> 366633 INFO  (TEST-DeleteReplicaTest.deleteReplicaOnIndexing-seed#[B93C9E584072689C]) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 4 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.DeleteReplicaTest_B93C9E584072689C-001/tempDir-001
   [junit4]   2> 366633 INFO  (TEST-DeleteReplicaTest.deleteReplicaOnIndexing-seed#[B93C9E584072689C]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 366634 INFO  (ZkTestServer Run Thread) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 366634 INFO  (ZkTestServer Run Thread) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 366734 INFO  (TEST-DeleteReplicaTest.deleteReplicaOnIndexing-seed#[B93C9E584072689C]) [    ] o.a.s.c.ZkTestServer start zk server on port:35194
   [junit4]   2> 366734 INFO  (TEST-DeleteReplicaTest.deleteReplicaOnIndexing-seed#[B93C9E584072689C]) [    ] o.a.s.c.ZkTestServer parse host and port list: localhost:35194
   [junit4]   2> 366734 INFO  (TEST-DeleteReplicaTest.deleteReplicaOnIndexing-seed#[B93C9E584072689C]) [    ] o.a.s.c.ZkTestServer connecting to localhost 35194
   [junit4]   2> 366744 INFO  (zkConnectionManagerCallback-2362-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 366750 INFO  (zkConnectionManagerCallback-2364-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 366752 INFO  (zkConnectionManagerCallback-2366-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 366756 WARN  (jetty-launcher-2359-thread-2) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 366757 WARN  (jetty-launcher-2359-thread-1) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 366758 INFO  (jetty-launcher-2359-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 366758 INFO  (jetty-launcher-2359-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 366758 INFO  (jetty-launcher-2359-thread-1) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 366763 INFO  (jetty-launcher-2359-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 366763 INFO  (jetty-launcher-2359-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 366763 INFO  (jetty-launcher-2359-thread-2) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 366766 INFO  (jetty-launcher-2359-thread-1) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 366766 INFO  (jetty-launcher-2359-thread-1) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 366766 INFO  (jetty-launcher-2359-thread-1) [    ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 366768 INFO  (jetty-launcher-2359-thread-2) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 366768 INFO  (jetty-launcher-2359-thread-2) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 366768 INFO  (jetty-launcher-2359-thread-2) [    ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 366769 INFO  (jetty-launcher-2359-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3f7c8043{/solr,null,AVAILABLE}
   [junit4]   2> 366769 INFO  (jetty-launcher-2359-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@754adf5a{/solr,null,AVAILABLE}
   [junit4]   2> 366769 WARN  (jetty-launcher-2359-thread-2) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@27062d5c[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 366769 WARN  (jetty-launcher-2359-thread-1) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@2eea6638[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 366770 INFO  (jetty-launcher-2359-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@54270893{SSL,[ssl, http/1.1]}{127.0.0.1:42891}
   [junit4]   2> 366770 INFO  (jetty-launcher-2359-thread-2) [    ] o.e.j.s.Server Started @366825ms
   [junit4]   2> 366770 INFO  (jetty-launcher-2359-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=42891}
   [junit4]   2> 366771 ERROR (jetty-launcher-2359-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 366771 INFO  (jetty-launcher-2359-thread-2) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 366771 INFO  (jetty-launcher-2359-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 366771 INFO  (jetty-launcher-2359-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 366771 INFO  (jetty-launcher-2359-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 366771 INFO  (jetty-launcher-2359-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-12-11T15:36:16.244Z
   [junit4]   2> 366777 INFO  (jetty-launcher-2359-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@7ffe97af{SSL,[ssl, http/1.1]}{127.0.0.1:36678}
   [junit4]   2> 366777 INFO  (jetty-launcher-2359-thread-1) [    ] o.e.j.s.Server Started @366832ms
   [junit4]   2> 366778 INFO  (jetty-launcher-2359-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=36678}
   [junit4]   2> 366778 ERROR (jetty-launcher-2359-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 366778 INFO  (jetty-launcher-2359-thread-1) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 366778 INFO  (jetty-launcher-2359-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 366778 INFO  (jetty-launcher-2359-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 366778 INFO  (jetty-launcher-2359-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 366778 INFO  (jetty-launcher-2359-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-12-11T15:36:16.251Z
   [junit4]   2> 366780 INFO  (zkConnectionManagerCallback-2370-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 366783 INFO  (jetty-launcher-2359-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 366797 INFO  (zkConnectionManagerCallback-2368-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 366797 INFO  (jetty-launcher-2359-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 366800 WARN  (jetty-launcher-2359-thread-4) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 366800 INFO  (jetty-launcher-2359-thread-4) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 366800 INFO  (jetty-launcher-2359-thread-4) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 366800 INFO  (jetty-launcher-2359-thread-4) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 366805 WARN  (jetty-launcher-2359-thread-3) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 366806 INFO  (jetty-launcher-2359-thread-3) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 366806 INFO  (jetty-launcher-2359-thread-3) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 366806 INFO  (jetty-launcher-2359-thread-3) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 366838 INFO  (jetty-launcher-2359-thread-4) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 366838 INFO  (jetty-launcher-2359-thread-4) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 366838 INFO  (jetty-launcher-2359-thread-4) [    ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 366842 INFO  (jetty-launcher-2359-thread-4) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@217d25bc{/solr,null,AVAILABLE}
   [junit4]   2> 366842 WARN  (jetty-launcher-2359-thread-4) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@7bd892d4[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 366852 INFO  (jetty-launcher-2359-thread-3) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 366852 INFO  (jetty-launcher-2359-thread-3) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 366853 INFO  (jetty-launcher-2359-thread-3) [    ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 366853 INFO  (jetty-launcher-2359-thread-3) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@68b204e3{/solr,null,AVAILABLE}
   [junit4]   2> 366854 WARN  (jetty-launcher-2359-thread-3) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@376d81b5[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 366855 INFO  (jetty-launcher-2359-thread-3) [    ] o.e.j.s.AbstractConnector Started ServerConnector@ac156d5{SSL,[ssl, http/1.1]}{127.0.0.1:40658}
   [junit4]   2> 366855 INFO  (jetty-launcher-2359-thread-3) [    ] o.e.j.s.Server Started @366910ms
   [junit4]   2> 366855 INFO  (jetty-launcher-2359-thread-3) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=40658}
   [junit4]   2> 366855 ERROR (jetty-launcher-2359-thread-3) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 366855 INFO  (jetty-launcher-2359-thread-3) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 366855 INFO  (jetty-launcher-2359-thread-3) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 366855 INFO  (jetty-launcher-2359-thread-3) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 366855 INFO  (jetty-launcher-2359-thread-3) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 366855 INFO  (jetty-launcher-2359-thread-4) [    ] o.e.j.s.AbstractConnector Started ServerConnector@66baa331{SSL,[ssl, http/1.1]}{127.0.0.1:35241}
   [junit4]   2> 366855 INFO  (jetty-launcher-2359-thread-3) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-12-11T15:36:16.328Z
   [junit4]   2> 366855 INFO  (jetty-launcher-2359-thread-4) [    ] o.e.j.s.Server Started @366910ms
   [junit4]   2> 366855 INFO  (jetty-launcher-2359-thread-4) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=35241}
   [junit4]   2> 366856 ERROR (jetty-launcher-2359-thread-4) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 366856 INFO  (jetty-launcher-2359-thread-4) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 366856 INFO  (jetty-launcher-2359-thread-4) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 366856 INFO  (jetty-launcher-2359-thread-4) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 366856 INFO  (jetty-launcher-2359-thread-4) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 366856 INFO  (jetty-launcher-2359-thread-4) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-12-11T15:36:16.329Z
   [junit4]   2> 366890 INFO  (zkConnectionManagerCallback-2372-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 366892 INFO  (jetty-launcher-2359-thread-3) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 366896 INFO  (zkConnectionManagerCallback-2374-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 366897 INFO  (jetty-launcher-2359-thread-4) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 366939 INFO  (jetty-launcher-2359-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=localhost:35194/solr
   [junit4]   2> 366948 INFO  (zkConnectionManagerCallback-2378-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 366980 INFO  (zkConnectionManagerCallback-2380-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 366999 INFO  (jetty-launcher-2359-thread-4) [    ] o.a.s.c.ZkContainer Zookeeper client=localhost:35194/solr
   [junit4]   2> 367025 INFO  (zkConnectionManagerCallback-2386-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 367056 INFO  (zkConnectionManagerCallback-2388-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 367061 INFO  (jetty-launcher-2359-thread-3) [    ] o.a.s.c.ZkContainer Zookeeper client=localhost:35194/solr
   [junit4]   2> 367063 INFO  (zkConnectionManagerCallback-2394-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 367065 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x10152658eff000b, likely client has closed socket
   [junit4]   2> 367069 INFO  (zkConnectionManagerCallback-2396-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 367074 INFO  (jetty-launcher-2359-thread-4) [n:127.0.0.1:35241_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:35241_solr
   [junit4]   2> 367080 INFO  (jetty-launcher-2359-thread-4) [n:127.0.0.1:35241_solr    ] o.a.s.c.Overseer Overseer (id=72429665158889482-127.0.0.1:35241_solr-n_0000000000) starting
   [junit4]   2> 367091 INFO  (jetty-launcher-2359-thread-3) [n:127.0.0.1:40658_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40658_solr
   [junit4]   2> 367108 INFO  (zkCallback-2395-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 367115 INFO  (jetty-launcher-2359-thread-2) [n:127.0.0.1:42891_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 367121 INFO  (zkConnectionManagerCallback-2403-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 367122 INFO  (zkCallback-2387-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 367123 INFO  (jetty-launcher-2359-thread-4) [n:127.0.0.1:35241_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 367124 INFO  (jetty-launcher-2359-thread-2) [n:127.0.0.1:42891_solr    ] o.a.s.c.ZkController Publish node=127.0.0.1:42891_solr as DOWN
   [junit4]   2> 367124 INFO  (jetty-launcher-2359-thread-4) [n:127.0.0.1:35241_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at localhost:35194/solr ready
   [junit4]   2> 367125 INFO  (jetty-launcher-2359-thread-2) [n:127.0.0.1:42891_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 367125 INFO  (jetty-launcher-2359-thread-2) [n:127.0.0.1:42891_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:42891_solr
   [junit4]   2> 367131 INFO  (OverseerStateUpdate-72429665158889482-127.0.0.1:35241_solr-n_0000000000) [n:127.0.0.1:35241_solr    ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:35241_solr
   [junit4]   2> 367132 INFO  (OverseerStateUpdate-72429665158889482-127.0.0.1:35241_solr-n_0000000000) [n:127.0.0.1:35241_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 367137 INFO  (jetty-launcher-2359-thread-4) [n:127.0.0.1:35241_solr    ] o.a.s.c.ZkController Publish node=127.0.0.1:35241_solr as DOWN
   [junit4]   2> 367137 INFO  (zkCallback-2395-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 367143 INFO  (zkCallback-2402-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 367144 INFO  (jetty-launcher-2359-thread-4) [n:127.0.0.1:35241_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 367144 INFO  (jetty-launcher-2359-thread-4) [n:127.0.0.1:35241_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35241_solr
   [junit4]   2> 367151 INFO  (zkCallback-2387-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 367151 INFO  (zkCallback-2395-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 367151 INFO  (zkCallback-2379-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (3)
   [junit4]   2> 367160 INFO  (zkCallback-2402-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 367171 INFO  (OverseerAutoScalingTriggerThread-72429665158889482-127.0.0.1:35241_solr-n_0000000000) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:40658_solr, 127.0.0.1:35241_solr, 127.0.0.1:42891_solr]
   [junit4]   2> 367171 INFO  (jetty-launcher-2359-thread-4) [n:127.0.0.1:35241_solr    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 367191 INFO  (ScheduledTrigger-1952-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
   [junit4]   2> 367192 INFO  (zkConnectionManagerCallback-2413-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 367193 INFO  (jetty-launcher-2359-thread-2) [n:127.0.0.1:42891_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 367194 INFO  (jetty-launcher-2359-thread-2) [n:127.0.0.1:42891_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at localhost:35194/solr ready
   [junit4]   2> 367196 INFO  (zkConnectionManagerCallback-2408-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 367198 INFO  (jetty-launcher-2359-thread-3) [n:127.0.0.1:40658_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 367199 INFO  (jetty-launcher-2359-thread-3) [n:127.0.0.1:40658_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at localhost:35194/solr ready
   [junit4]   2> 367206 INFO  (jetty-launcher-2359-thread-2) [n:127.0.0.1:42891_solr    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 367213 INFO  (jetty-launcher-2359-thread-3) [n:127.0.0.1:40658_solr    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 367239 INFO  (jetty-launcher-2359-thread-4) [n:127.0.0.1:35241_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35241.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@114de92a
   [junit4]   2> 367276 INFO  (jetty-launcher-2359-thread-2) [n:127.0.0.1:42891_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_42891.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@114de92a
   [junit4]   2> 367277 INFO  (jetty-launcher-2359-thread-3) [n:127.0.0.1:40658_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40658.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@114de92a
   [junit4]   2> 367295 INFO  (jetty-launcher-2359-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=localhost:35194/solr
   [junit4]   2> 367299 INFO  (jetty-launcher-2359-thread-2) [n:127.0.0.1:42891_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_42891.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@114de92a
   [junit4]   2> 367299 INFO  (jetty-launcher-2359-thread-2) [n:127.0.0.1:42891_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_42891.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@114de92a
   [junit4]   2> 367301 INFO  (jetty-launcher-2359-thread-2) [n:127.0.0.1:42891_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.DeleteReplicaTest_B93C9E584072689C-001/tempDir-001/node2/.
   [junit4]   2> 367310 INFO  (jetty-launcher-2359-thread-4) [n:127.0.0.1:35241_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35241.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@114de92a
   [junit4]   2> 367310 INFO  (jetty-launcher-2359-thread-4) [n:127.0.0.1:35241_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35241.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@114de92a
   [junit4]   2> 367312 INFO  (jetty-launcher-2359-thread-4) [n:127.0.0.1:35241_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.DeleteReplicaTest_B93C9E584072689C-001/tempDir-001/node4/.
   [junit4]   2> 367314 INFO  (jetty-launcher-2359-thread-3) [n:127.0.0.1:40658_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40658.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@114de92a
   [junit4]   2> 367314 INFO  (jetty-launcher-2359-thread-3) [n:127.0.0.1:40658_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40658.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@114de92a
   [junit4]   2> 367316 INFO  (jetty-launcher-2359-thread-3) [n:127.0.0.1:40658_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.DeleteReplicaTest_B93C9E584072689C-001/tempDir-001/node3/.
   [junit4]   2> 367354 INFO  (zkConnectionManagerCallback-2420-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 367390 INFO  (zkConnectionManagerCallback-2422-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 367396 INFO  (jetty-launcher-2359-thread-1) [n:127.0.0.1:36678_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 367400 INFO  (jetty-launcher-2359-thread-1) [n:127.0.0.1:36678_solr    ] o.a.s.c.ZkController Publish node=127.0.0.1:36678_solr as DOWN
   [junit4]   2> 367401 INFO  (jetty-launcher-2359-thread-1) [n:127.0.0.1:36678_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 367401 INFO  (jetty-launcher-2359-thread-1) [n:127.0.0.1:36678_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:36678_solr
   [junit4]   2> 367403 INFO  (zkCallback-2395-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 367403 INFO  (zkCallback-2387-thread-2) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 367403 INFO  (zkCallback-2402-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 367421 INFO  (zkCallback-2379-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 367421 INFO  (zkCallback-2412-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 367422 INFO  (zkCallback-2421-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 367425 INFO  (zkCallback-2407-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 367488 INFO  (zkConnectionManagerCallback-2429-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 367490 INFO  (jetty-launcher-2359-thread-1) [n:127.0.0.1:36678_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 367491 INFO  (jetty-launcher-2359-thread-1) [n:127.0.0.1:36678_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at localhost:35194/solr ready
   [junit4]   2> 367492 INFO  (jetty-launcher-2359-thread-1) [n:127.0.0.1:36678_solr    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 367519 INFO  (jetty-launcher-2359-thread-1) [n:127.0.0.1:36678_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_36678.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@114de92a
   [junit4]   2> 367534 INFO  (jetty-launcher-2359-thread-1) [n:127.0.0.1:36678_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_36678.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@114de92a
   [junit4]   2> 367534 INFO  (jetty-launcher-2359-thread-1) [n:127.0.0.1:36678_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_36678.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@114de92a
   [junit4]   2> 367536 INFO  (jetty-launcher-2359-thread-1) [n:127.0.0.1:36678_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.DeleteReplicaTest_B93C9E584072689C-001/tempDir-001/node1/.
   [junit4]   2> 367608 INFO  (zkConnectionManagerCallback-2435-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 367609 INFO  (TEST-DeleteReplicaTest.deleteReplicaOnIndexing-seed#[B93C9E584072689C]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 367611 INFO  (TEST-DeleteReplicaTest.deleteReplicaOnIndexing-seed#[B93C9E584072689C]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at localhost:35194/solr ready
   [junit4]   2> 367644 INFO  (qtp1023246246-6306) [n:127.0.0.1:40658_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf&name=deleteReplicaOnIndexing&nrtReplicas=2&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 367648 INFO  (OverseerThreadFactory-1954-thread-1-processing-n:127.0.0.1:35241_solr) [n:127.0.0.1:35241_solr    ] o.a.s.c.a.c.CreateCollectionCmd Create collection deleteReplicaOnIndexing
   [junit4]   2> 367766 INFO  (qtp1023246246-6307) [n:127.0.0.1:40658_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 367770 INFO  (qtp1023246246-6308) [n:127.0.0.1:40658_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 367771 INFO  (qtp1023246246-6309) [n:127.0.0.1:40658_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 367772 INFO  (qtp1023246246-6305) [n:127.0.0.1:40658_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 367773 INFO  (qtp1023246246-6307) [n:127.0.0.1:40658_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 367818 INFO  (qtp1352736580-6297) [n:127.0.0.1:35241_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 367822 INFO  (qtp1352736580-6298) [n:127.0.0.1:35241_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 367823 INFO  (qtp1352736580-6299) [n:127.0.0.1:35241_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 367824 INFO  (qtp1352736580-6300) [n:127.0.0.1:35241_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 367825 INFO  (qtp1352736580-6296) [n:127.0.0.1:35241_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 367849 INFO  (qtp444347843-6277) [n:127.0.0.1:36678_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 367851 INFO  (qtp444347843-6273) [n:127.0.0.1:36678_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 367857 INFO  (qtp444347843-6274) [n:127.0.0.1:36678_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 367858 INFO  (qtp444347843-6275) [n:127.0.0.1:36678_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 367859 INFO  (qtp444347843-6276) [n:127.0.0.1:36678_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 367886 INFO  (qtp825239796-6285) [n:127.0.0.1:42891_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 367888 INFO  (qtp825239796-6281) [n:127.0.0.1:42891_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 367889 INFO  (qtp825239796-6282) [n:127.0.0.1:42891_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 367890 INFO  (qtp825239796-6283) [n:127.0.0.1:42891_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 367891 INFO  (qtp825239796-6284) [n:127.0.0.1:42891_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 367901 INFO  (OverseerStateUpdate-72429665158889482-127.0.0.1:35241_solr-n_0000000000) [n:127.0.0.1:35241_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"deleteReplicaOnIndexing",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"deleteReplicaOnIndexing_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:42891/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 367905 INFO  (OverseerStateUpdate-72429665158889482-127.0.0.1:35241_solr-n_0000000000) [n:127.0.0.1:35241_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"deleteReplicaOnIndexing",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"deleteReplicaOnIndexing_shard1_replica_n2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:36678/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 368150 INFO  (qtp444347843-6277) [n:127.0.0.1:36678_solr    x:deleteReplicaOnIndexing_shard1_replica_n2] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=deleteReplicaOnIndexing_shard1_replica_n2&action=CREATE&numShards=1&collection=deleteReplicaOnIndexing&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 368151 INFO  (qtp825239796-6285) [n:127.0.0.1:42891_solr    x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=deleteReplicaOnIndexing_shard1_replica_n1&action=CREATE&numShards=1&collection=deleteReplicaOnIndexing&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 368193 INFO  (ScheduledTrigger-1952-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 and last live nodes: 3
   [junit4]   2> 369175 INFO  (qtp444347843-6277) [n:127.0.0.1:36678_solr c:deleteReplicaOnIndexing s:shard1 r:core_node4 x:deleteReplicaOnIndexing_shard1_replica_n2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 369175 INFO  (qtp825239796-6285) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 369196 INFO  (ScheduledTrigger-1952-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 and last live nodes: 4
   [junit4]   2> 369201 INFO  (qtp444347843-6277) [n:127.0.0.1:36678_solr c:deleteReplicaOnIndexing s:shard1 r:core_node4 x:deleteReplicaOnIndexing_shard1_replica_n2] o.a.s.s.IndexSchema [deleteReplicaOnIndexing_shard1_replica_n2] Schema name=minimal
   [junit4]   2> 369201 INFO  (qtp825239796-6285) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.s.IndexSchema [deleteReplicaOnIndexing_shard1_replica_n1] Schema name=minimal
   [junit4]   2> 369205 INFO  (qtp444347843-6277) [n:127.0.0.1:36678_solr c:deleteReplicaOnIndexing s:shard1 r:core_node4 x:deleteReplicaOnIndexing_shard1_replica_n2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 369205 INFO  (qtp444347843-6277) [n:127.0.0.1:36678_solr c:deleteReplicaOnIndexing s:shard1 r:core_node4 x:deleteReplicaOnIndexing_shard1_replica_n2] o.a.s.c.CoreContainer Creating SolrCore 'deleteReplicaOnIndexing_shard1_replica_n2' using configuration from collection deleteReplicaOnIndexing, trusted=true
   [junit4]   2> 369205 INFO  (qtp444347843-6277) [n:127.0.0.1:36678_solr c:deleteReplicaOnIndexing s:shard1 r:core_node4 x:deleteReplicaOnIndexing_shard1_replica_n2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_36678.solr.core.deleteReplicaOnIndexing.shard1.replica_n2' (registry 'solr.core.deleteReplicaOnIndexing.shard1.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@114de92a
   [junit4]   2> 369206 INFO  (qtp444347843-6277) [n:127.0.0.1:36678_solr c:deleteReplicaOnIndexing s:shard1 r:core_node4 x:deleteReplicaOnIndexing_shard1_replica_n2] o.a.s.c.SolrCore [[deleteReplicaOnIndexing_shard1_replica_n2] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.DeleteReplicaTest_B93C9E584072689C-001/tempDir-001/node1/deleteReplicaOnIndexing_shard1_replica_n2], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.DeleteReplicaTest_B93C9E584072689C-001/tempDir-001/node1/./deleteReplicaOnIndexing_shard1_replica_n2/data/]
   [junit4]   2> 369206 INFO  (qtp825239796-6285) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 369206 INFO  (qtp825239796-6285) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'deleteReplicaOnIndexing_shard1_replica_n1' using configuration from collection deleteReplicaOnIndexing, trusted=true
   [junit4]   2> 369206 INFO  (qtp825239796-6285) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_42891.solr.core.deleteReplicaOnIndexing.shard1.replica_n1' (registry 'solr.core.deleteReplicaOnIndexing.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@114de92a
   [junit4]   2> 369207 INFO  (qtp825239796-6285) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.c.SolrCore [[deleteReplicaOnIndexing_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.DeleteReplicaTest_B93C9E584072689C-001/tempDir-001/node2/deleteReplicaOnIndexing_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.DeleteReplicaTest_B93C9E584072689C-001/tempDir-001/node2/./deleteReplicaOnIndexing_shard1_replica_n1/data/]
   [junit4]   2> 369307 INFO  (qtp825239796-6285) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 369307 INFO  (qtp825239796-6285) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 369307 INFO  (qtp444347843-6277) [n:127.0.0.1:36678_solr c:deleteReplicaOnIndexing s:shard1 r:core_node4 x:deleteReplicaOnIndexing_shard1_replica_n2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 369307 INFO  (qtp444347843-6277) [n:127.0.0.1:36678_solr c:deleteReplicaOnIndexing s:shard1 r:core_node4 x:deleteReplicaOnIndexing_shard1_replica_n2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 369311 INFO  (qtp444347843-6277) [n:127.0.0.1:36678_solr c:deleteReplicaOnIndexing s:shard1 r:core_node4 x:deleteReplicaOnIndexing_shard1_replica_n2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 369311 INFO  (qtp825239796-6285) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 369311 INFO  (qtp825239796-6285) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 369311 INFO  (qtp444347843-6277) [n:127.0.0.1:36678_solr c:deleteReplicaOnIndexing s:shard1 r:core_node4 x:deleteReplicaOnIndexing_shard1_replica_n2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 369313 INFO  (qtp825239796-6285) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3f772dcf[deleteReplicaOnIndexing_shard1_replica_n1] main]
   [junit4]   2> 369313 INFO  (qtp444347843-6277) [n:127.0.0.1:36678_solr c:deleteReplicaOnIndexing s:shard1 r:core_node4 x:deleteReplicaOnIndexing_shard1_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@42ae07[deleteReplicaOnIndexing_shard1_replica_n2] main]
   [junit4]   2> 369315 INFO  (qtp825239796-6285) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 369315 INFO  (qtp444347843-6277) [n:127.0.0.1:36678_solr c:deleteReplicaOnIndexing s:shard1 r:core_node4 x:deleteReplicaOnIndexing_shard1_replica_n2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 369316 INFO  (qtp825239796-6285) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 369316 INFO  (qtp444347843-6277) [n:127.0.0.1:36678_solr c:deleteReplicaOnIndexing s:shard1 r:core_node4 x:deleteReplicaOnIndexing_shard1_replica_n2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 369317 INFO  (qtp825239796-6285) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 369317 INFO  (qtp444347843-6277) [n:127.0.0.1:36678_solr c:deleteReplicaOnIndexing s:shard1 r:core_node4 x:deleteReplicaOnIndexing_shard1_replica_n2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 369317 INFO  (qtp825239796-6285) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1619570279097303040
   [junit4]   2> 369317 INFO  (qtp444347843-6277) [n:127.0.0.1:36678_solr c:deleteReplicaOnIndexing s:shard1 r:core_node4 x:deleteReplicaOnIndexing_shard1_replica_n2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1619570279097303040
   [junit4]   2> 369322 INFO  (searcherExecutor-1975-thread-1-processing-n:127.0.0.1:42891_solr x:deleteReplicaOnIndexing_shard1_replica_n1 c:deleteReplicaOnIndexing s:shard1 r:core_node3) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.c.SolrCore [deleteReplicaOnIndexing_shard1_replica_n1] Registered new searcher Searcher@3f772dcf[deleteReplicaOnIndexing_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 369326 INFO  (searcherExecutor-1974-thread-1-processing-n:127.0.0.1:36678_solr x:deleteReplicaOnIndexing_shard1_replica_n2 c:deleteReplicaOnIndexing s:shard1 r:core_node4) [n:127.0.0.1:36678_solr c:deleteReplicaOnIndexing s:shard1 r:core_node4 x:deleteReplicaOnIndexing_shard1_replica_n2] o.a.s.c.SolrCore [deleteReplicaOnIndexing_shard1_replica_n2] Registered new searcher Searcher@42ae07[deleteReplicaOnIndexing_shard1_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 369326 INFO  (qtp444347843-6277) [n:127.0.0.1:36678_solr c:deleteReplicaOnIndexing s:shard1 r:core_node4 x:deleteReplicaOnIndexing_shard1_replica_n2] o.a.s.c.ZkShardTerms Failed to save terms, version is not a match, retrying
   [junit4]   2> 369327 INFO  (qtp825239796-6285) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.c.ZkShardTerms Successful update of terms at /collections/deleteReplicaOnIndexing/terms/shard1 to Terms{values={core_node3=0}, version=0}
   [junit4]   2> 369327 INFO  (qtp825239796-6285) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/deleteReplicaOnIndexing/leaders/shard1
   [junit4]   2> 369327 INFO  (qtp444347843-6277) [n:127.0.0.1:36678_solr c:deleteReplicaOnIndexing s:shard1 r:core_node4 x:deleteReplicaOnIndexing_shard1_replica_n2] o.a.s.c.ZkShardTerms Successful update of terms at /collections/deleteReplicaOnIndexing/terms/shard1 to Terms{values={core_node3=0, core_node4=0}, version=1}
   [junit4]   2> 369328 INFO  (qtp444347843-6277) [n:127.0.0.1:36678_solr c:deleteReplicaOnIndexing s:shard1 r:core_node4 x:deleteReplicaOnIndexing_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/deleteReplicaOnIndexing/leaders/shard1
   [junit4]   2> 369333 INFO  (qtp825239796-6285) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 369333 INFO  (qtp825239796-6285) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 369333 INFO  (qtp825239796-6285) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:42891/solr/deleteReplicaOnIndexing_shard1_replica_n1/
   [junit4]   2> 369334 INFO  (qtp825239796-6285) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.u.PeerSync PeerSync: core=deleteReplicaOnIndexing_shard1_replica_n1 url=https://127.0.0.1:42891/solr START replicas=[https://127.0.0.1:36678/solr/deleteReplicaOnIndexing_shard1_replica_n2/] nUpdates=100
   [junit4]   2> 369334 INFO  (qtp825239796-6285) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.u.PeerSync PeerSync: core=deleteReplicaOnIndexing_shard1_replica_n1 url=https://127.0.0.1:42891/solr DONE.  We have no versions.  sync failed.
   [junit4]   2> 369342 INFO  (qtp444347843-6275) [n:127.0.0.1:36678_solr c:deleteReplicaOnIndexing s:shard1 r:core_node4 x:deleteReplicaOnIndexing_shard1_replica_n2] o.a.s.c.S.Request [deleteReplicaOnIndexing_shard1_replica_n2]  webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=1
   [junit4]   2> 369343 INFO  (qtp825239796-6285) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
   [junit4]   2> 369343 INFO  (qtp825239796-6285) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 369343 INFO  (qtp825239796-6285) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/deleteReplicaOnIndexing/leaders/shard1/leader after winning as /collections/deleteReplicaOnIndexing/leader_elect/shard1/election/72429665158889480-core_node3-n_0000000000
   [junit4]   2> 369345 INFO  (qtp825239796-6285) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:42891/solr/deleteReplicaOnIndexing_shard1_replica_n1/ shard1
   [junit4]   2> 369448 INFO  (zkCallback-2379-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteReplicaOnIndexing/state.json] for collection [deleteReplicaOnIndexing] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 369448 INFO  (zkCallback-2379-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteReplicaOnIndexing/state.json] for collection [deleteReplicaOnIndexing] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 369449 INFO  (qtp825239796-6285) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 369452 INFO  (qtp825239796-6285) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=deleteReplicaOnIndexing_shard1_replica_n1&action=CREATE&numShards=1&collection=deleteReplicaOnIndexing&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1300
   [junit4]   2> 369551 INFO  (zkCallback-2379-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteReplicaOnIndexing/state.json] for collection [deleteReplicaOnIndexing] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 369552 INFO  (zkCallback-2379-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteReplicaOnIndexing/state.json] for collection [deleteReplicaOnIndexing] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 369552 INFO  (zkCallback-2379-thread-3) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteReplicaOnIndexing/state.json] for collection [deleteReplicaOnIndexing] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 370196 INFO  (ScheduledTrigger-1952-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 and last live nodes: 4
   [junit4]   2> 370336 INFO  (qtp444347843-6277) [n:127.0.0.1:36678_solr c:deleteReplicaOnIndexing s:shard1 r:core_node4 x:deleteReplicaOnIndexing_shard1_replica_n2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=deleteReplicaOnIndexing_shard1_replica_n2&action=CREATE&numShards=1&collection=deleteReplicaOnIndexing&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2186
   [junit4]   2> 370350 INFO  (qtp1023246246-6306) [n:127.0.0.1:40658_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
   [junit4]   2> 370436 INFO  (zkCallback-2395-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteReplicaOnIndexing/state.json] for collection [deleteReplicaOnIndexing] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 370437 INFO  (zkCallback-2379-thread-3) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteReplicaOnIndexing/state.json] for collection [deleteReplicaOnIndexing] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 370437 INFO  (zkCallback-2379-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteReplicaOnIndexing/state.json] for collection [deleteReplicaOnIndexing] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 370437 INFO  (zkCallback-2379-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteReplicaOnIndexing/state.json] for collection [deleteReplicaOnIndexing] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 370437 INFO  (zkCallback-2421-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteReplicaOnIndexing/state.json] for collection [deleteReplicaOnIndexing] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 370437 INFO  (zkCallback-2421-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteReplicaOnIndexing/state.json] for collection [deleteReplicaOnIndexing] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 370438 INFO  (qtp1023246246-6306) [n:127.0.0.1:40658_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf&name=deleteReplicaOnIndexing&nrtReplicas=2&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=2793
   [junit4]   2> 370440 INFO  (TEST-DeleteReplicaTest.deleteReplicaOnIndexing-seed#[B93C9E584072689C]) [    ] o.a.s.c.SolrCloudTestCase active replica count: 2 expected replica count: 2
   [junit4]   2> 370678 INFO  (qtp1023246246-6308) [n:127.0.0.1:40658_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :deletereplica with params replica=core_node4&action=DELETEREPLICA&collection=deleteReplicaOnIndexing&shard=shard1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 370691 INFO  (qtp825239796-6285) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.c.ZkShardTerms Successful update of terms at /collections/deleteReplicaOnIndexing/terms/shard1 to Terms{values={core_node3=1, core_node4=1}, version=2}
   [junit4]   2> 370713 INFO  (OverseerCollectionConfigSetProcessor-72429665158889482-127.0.0.1:35241_solr-n_0000000000) [n:127.0.0.1:35241_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 370744 INFO  (qtp444347843-6277) [n:127.0.0.1:36678_solr c:deleteReplicaOnIndexing s:shard1 r:core_node4 x:deleteReplicaOnIndexing_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [deleteReplicaOnIndexing_shard1_replica_n2]  webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:42891/solr/deleteReplicaOnIndexing_shard1_replica_n1/&wt=javabin&version=2}{add=[30000 (1619570280332525568)]} 0 33
   [junit4]   2> 370751 INFO  (qtp825239796-6284) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [deleteReplicaOnIndexing_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[30000 (1619570280332525568)]} 0 256
   [junit4]   2> 370756 INFO  (qtp444347843-6276) [n:127.0.0.1:36678_solr c:deleteReplicaOnIndexing s:shard1 r:core_node4 x:deleteReplicaOnIndexing_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [deleteReplicaOnIndexing_shard1_replica_n2]  webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:42891/solr/deleteReplicaOnIndexing_shard1_replica_n1/&wt=javabin&version=2}{add=[0 (1619570280333574144)]} 0 45
   [junit4]   2> 370756 INFO  (qtp444347843-6273) [n:127.0.0.1:36678_solr c:deleteReplicaOnIndexing s:shard1 r:core_node4 x:deleteReplicaOnIndexing_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [deleteReplicaOnIndexing_shard1_replica_n2]  webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:42891/solr/deleteReplicaOnIndexing_shard1_replica_n1/&wt=javabin&version=2}{add=[50000 (1619570280523366400)]} 0 45
   [junit4]   2> 370763 INFO  (qtp825239796-6282) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [deleteReplicaOnIndexing_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[50000 (1619570280523366400)]} 0 86
   [junit4]   2> 370767 INFO  (qtp825239796-6285) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [deleteReplicaOnIndexing_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[0 (1619570280333574144)]} 0 271
   [junit4]   2> 370847 INFO  (qtp444347843-6277) [n:127.0.0.1:36678_solr c:deleteReplicaOnIndexing s:shard1 r:core_node4 x:deleteReplicaOnIndexing_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [deleteReplicaOnIndexing_shard1_replica_n2]  webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:42891/solr/deleteReplicaOnIndexing_shard1_replica_n1/&wt=javabin&version=2}{add=[40000 (1619570280327282688)]} 0 30
   [junit4]   2> 370849 INFO  (qtp825239796-6281) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [deleteReplicaOnIndexing_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[40000 (1619570280327282688)]} 0 359
   [junit4]   2> 370850 INFO  (qtp444347843-6276) [n:127.0.0.1:36678_solr    x:deleteReplicaOnIndexing_shard1_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.deleteReplicaOnIndexing.shard1.replica_n2, tag=null
   [junit4]   2> 370850 INFO  (qtp444347843-6276) [n:127.0.0.1:36678_solr    x:deleteReplicaOnIndexing_shard1_replica_n2] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@b764266: rootName = solr_36678, domain = solr.core.deleteReplicaOnIndexing.shard1.replica_n2, service url = null, agent id = null] for registry solr.core.deleteReplicaOnIndexing.shard1.replica_n2 / com.codahale.metrics.MetricRegistry@5be511a
   [junit4]   2> 370950 WARN  (updateExecutor-2375-thread-5-processing-x:deleteReplicaOnIndexing_shard1_replica_n1 https:////127.0.0.1:36678//solr//deleteReplicaOnIndexing_shard1_replica_n2 r:core_node3 n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.c.s.i.ConcurrentUpdateSolrClient Failed to parse error response from https://127.0.0.1:36678/solr/deleteReplicaOnIndexing_shard1_replica_n2 due to: java.lang.RuntimeException: Invalid version (expected 2, but 60) or the data in not in 'javabin' format
   [junit4]   2> 370950 ERROR (updateExecutor-2375-thread-5-processing-x:deleteReplicaOnIndexing_shard1_replica_n1 https:////127.0.0.1:36678//solr//deleteReplicaOnIndexing_shard1_replica_n2 r:core_node3 n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.u.ErrorReportingConcurrentUpdateSolrClient error
   [junit4]   2> org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:36678/solr/deleteReplicaOnIndexing_shard1_replica_n2: Can not find: /solr/deleteReplicaOnIndexing_shard1_replica_n2/update
   [junit4]   2> 
   [junit4]   2> 
   [junit4]   2> 
   [junit4]   2> request: https://127.0.0.1:36678/solr/deleteReplicaOnIndexing_shard1_replica_n2/update?update.distrib=FROMLEADER&distrib.from=https%3A%2F%2F127.0.0.1%3A42891%2Fsolr%2FdeleteReplicaOnIndexing_shard1_replica_n1%2F&wt=javabin&version=2
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:385) ~[java/:?]
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:183) ~[java/:?]
   [junit4]   2> 	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176) ~[metrics-core-3.2.6.jar:3.2.6]
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) ~[java/:?]
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_191]
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_191]
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_191]
   [junit4]   2> 370959 WARN  (updateExecutor-2375-thread-2-processing-x:deleteReplicaOnIndexing_shard1_replica_n1 https:////127.0.0.1:36678//solr//deleteReplicaOnIndexing_shard1_replica_n2 r:core_node3 n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.c.s.i.ConcurrentUpdateSolrClient Failed to parse error response from https://127.0.0.1:36678/solr/deleteReplicaOnIndexing_shard1_replica_n2 due to: java.lang.RuntimeException: Invalid version (expected 2, but 60) or the data in not in 'javabin' format
   [junit4]   2> 370997 INFO  (qtp444347843-6276) [n:127.0.0.1:36678_solr    x:deleteReplicaOnIndexing_shard1_replica_n2] o.a.s.c.SolrCore [deleteReplicaOnIndexing_shard1_replica_n2]  CLOSING SolrCore org.apache.solr.core.SolrCore@55ab9149
   [junit4]   2> 370997 INFO  (qtp444347843-6276) [n:127.0.0.1:36678_solr    x:deleteReplicaOnIndexing_shard1_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.deleteReplicaOnIndexing.shard1.replica_n2, tag=55ab9149
   [junit4]   2> 370998 INFO  (qtp444347843-6276) [n:127.0.0.1:36678_solr    x:deleteReplicaOnIndexing_shard1_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.deleteReplicaOnIndexing.shard1.leader, tag=55ab9149
   [junit4]   2> 370998 INFO  (qtp444347843-6276) [n:127.0.0.1:36678_solr    x:deleteReplicaOnIndexing_shard1_replica_n2] o.a.s.u.DirectUpdateHandler2 Committing on IndexWriter close.
   [junit4]   2> 370998 INFO  (qtp444347843-6276) [n:127.0.0.1:36678_solr    x:deleteReplicaOnIndexing_shard1_replica_n2] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@44835b21 commitCommandVersion:0
   [junit4]   2> 370998 INFO  (qtp825239796-6283) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.u.SolrCmdDistributor SolrCmdDistributor found 1 errors
   [junit4]   2> 370998 ERROR (updateExecutor-2375-thread-2-processing-x:deleteReplicaOnIndexing_shard1_replica_n1 https:////127.0.0.1:36678//solr//deleteReplicaOnIndexing_shard1_replica_n2 r:core_node3 n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.u.ErrorReportingConcurrentUpdateSolrClient error
   [junit4]   2> org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:36678/solr/deleteReplicaOnIndexing_shard1_replica_n2: Can not find: /solr/deleteReplicaOnIndexing_shard1_replica_n2/update
   [junit4]   2> 
   [junit4]   2> 
   [junit4]   2> 
   [junit4]   2> request: https://127.0.0.1:36678/solr/deleteReplicaOnIndexing_shard1_replica_n2/update?update.distrib=FROMLEADER&distrib.from=https%3A%2F%2F127.0.0.1%3A42891%2Fsolr%2FdeleteReplicaOnIndexing_shard1_replica_n1%2F&wt=javabin&version=2
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:385) ~[java/:?]
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:183) ~[java/:?]
   [junit4]   2> 	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176) ~[metrics-core-3.2.6.jar:3.2.6]
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) ~[java/:?]
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_191]
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_191]
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_191]
   [junit4]   2> 371000 WARN  (updateExecutor-2375-thread-4-processing-x:deleteReplicaOnIndexing_shard1_replica_n1 https:////127.0.0.1:36678//solr//deleteReplicaOnIndexing_shard1_replica_n2 r:core_node3 n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.c.s.i.ConcurrentUpdateSolrClient Failed to parse error response from https://127.0.0.1:36678/solr/deleteReplicaOnIndexing_shard1_replica_n2 due to: java.lang.RuntimeException: Invalid version (expected 2, but 60) or the data in not in 'javabin' format
   [junit4]   2> 371001 ERROR (updateExecutor-2375-thread-4-processing-x:deleteReplicaOnIndexing_shard1_replica_n1 https:////127.0.0.1:36678//solr//deleteReplicaOnIndexing_shard1_replica_n2 r:core_node3 n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.u.ErrorReportingConcurrentUpdateSolrClient error
   [junit4]   2> org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:36678/solr/deleteReplicaOnIndexing_shard1_replica_n2: Can not find: /solr/deleteReplicaOnIndexing_shard1_replica_n2/update
   [junit4]   2> 
   [junit4]   2> 
   [junit4]   2> 
   [junit4]   2> request: https://127.0.0.1:36678/solr/deleteReplicaOnIndexing_shard1_replica_n2/update?update.distrib=FROMLEADER&distrib.from=https%3A%2F%2F127.0.0.1%3A42891%2Fsolr%2FdeleteReplicaOnIndexing_shard1_replica_n1%2F&wt=javabin&version=2
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:385) ~[java/:?]
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:183) ~[java/:?]
   [junit4]   2> 	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176) ~[metrics-core-3.2.6.jar:3.2.6]
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) ~[java/:?]
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_191]
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_191]
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_191]
   [junit4]   2> 371001 WARN  (updateExecutor-2375-thread-7-processing-x:deleteReplicaOnIndexing_shard1_replica_n1 https:////127.0.0.1:36678//solr//deleteReplicaOnIndexing_shard1_replica_n2 r:core_node3 n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.c.s.i.ConcurrentUpdateSolrClient Failed to parse error response from https://127.0.0.1:36678/solr/deleteReplicaOnIndexing_shard1_replica_n2 due to: java.lang.RuntimeException: Invalid version (expected 2, but 60) or the data in not in 'javabin' format
   [junit4]   2> 371001 ERROR (updateExecutor-2375-thread-7-processing-x:deleteReplicaOnIndexing_shard1_replica_n1 https:////127.0.0.1:36678//solr//deleteReplicaOnIndexing_shard1_replica_n2 r:core_node3 n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.u.ErrorReportingConcurrentUpdateSolrClient error
   [junit4]   2> org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:36678/solr/deleteReplicaOnIndexing_shard1_replica_n2: Can not find: /solr/deleteReplicaOnIndexing_shard1_replica_n2/update
   [junit4]   2> 
   [junit4]   2> 
   [junit4]   2> 
   [junit4]   2> request: https://127.0.0.1:36678/solr/deleteReplicaOnIndexing_shard1_replica_n2/update?update.distrib=FROMLEADER&distrib.from=https%3A%2F%2F127.0.0.1%3A42891%2Fsolr%2FdeleteReplicaOnIndexing_shard1_replica_n1%2F&wt=javabin&version=2
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:385) ~[java/:?]
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:183) ~[java/:?]
   [junit4]   2> 	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176) ~[metrics-core-3.2.6.jar:3.2.6]
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) ~[java/:?]
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_191]
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_191]
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_191]
   [junit4]   2> 371007 WARN  (updateExecutor-2375-thread-6-processing-x:deleteReplicaOnIndexing_shard1_replica_n1 https:////127.0.0.1:36678//solr//deleteReplicaOnIndexing_shard1_replica_n2 r:core_node3 n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.c.s.i.ConcurrentUpdateSolrClient Failed to parse error response from https://127.0.0.1:36678/solr/deleteReplicaOnIndexing_shard1_replica_n2 due to: java.lang.RuntimeException: Invalid version (expected 2, but 60) or the data in not in 'javabin' format
   [junit4]   2> 371007 ERROR (updateExecutor-2375-thread-6-processing-x:deleteReplicaOnIndexing_shard1_replica_n1 https:////127.0.0.1:36678//solr//deleteReplicaOnIndexing_shard1_replica_n2 r:core_node3 n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.u.ErrorReportingConcurrentUpdateSolrClient error
   [junit4]   2> org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:36678/solr/deleteReplicaOnIndexing_shard1_replica_n2: Can not find: /solr/deleteReplicaOnIndexing_shard1_replica_n2/update
   [junit4]   2> 
   [junit4]   2> 
   [junit4]   2> 
   [junit4]   2> request: https://127.0.0.1:36678/solr/deleteReplicaOnIndexing_shard1_replica_n2/update?update.distrib=FROMLEADER&distrib.from=https%3A%2F%2F127.0.0.1%3A42891%2Fsolr%2FdeleteReplicaOnIndexing_shard1_replica_n1%2F&wt=javabin&version=2
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:385) ~[java/:?]
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:183) ~[java/:?]
   [junit4]   2> 	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176) ~[metrics-core-3.2.6.jar:3.2.6]
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) ~[java/:?]
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_191]
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_191]
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_191]
   [junit4]   2> 371028 INFO  (qtp825239796-6626) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.u.SolrCmdDistributor SolrCmdDistributor found 1 errors
   [junit4]   2> 371028 INFO  (qtp825239796-6637) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.u.SolrCmdDistributor SolrCmdDistributor found 1 errors
   [junit4]   2> 371028 INFO  (qtp825239796-6636) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.u.SolrCmdDistributor SolrCmdDistributor found 1 errors
   [junit4]   2> 371028 INFO  (qtp825239796-6281) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.u.SolrCmdDistributor SolrCmdDistributor found 1 errors
   [junit4]   2> 371059 WARN  (updateExecutor-2375-thread-9-processing-x:deleteReplicaOnIndexing_shard1_replica_n1 https:////127.0.0.1:36678//solr//deleteReplicaOnIndexing_shard1_replica_n2 r:core_node3 n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.c.s.i.ConcurrentUpdateSolrClient Failed to parse error response from https://127.0.0.1:36678/solr/deleteReplicaOnIndexing_shard1_replica_n2 due to: java.lang.RuntimeException: Invalid version (expected 2, but 60) or the data in not in 'javabin' format
   [junit4]   2> 371059 ERROR (updateExecutor-2375-thread-9-processing-x:deleteReplicaOnIndexing_shard1_replica_n1 https:////127.0.0.1:36678//solr//deleteReplicaOnIndexing_shard1_replica_n2 r:core_node3 n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.u.ErrorReportingConcurrentUpdateSolrClient error
   [junit4]   2> org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:36678/solr/deleteReplicaOnIndexing_shard1_replica_n2: Can not find: /solr/deleteReplicaOnIndexing_shard1_replica_n2/update
   [junit4]   2> 
   [junit4]   2> 
   [junit4]   2> 
   [junit4]   2> request: https://127.0.0.1:36678/solr/deleteReplicaOnIndexing_shard1_replica_n2/update?update.distrib=FROMLEADER&distrib.from=https%3A%2F%2F127.0.0.1%3A42891%2Fsolr%2FdeleteReplicaOnIndexing_shard1_replica_n1%2F&wt=javabin&version=2
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:385) ~[java/:?]
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:183) ~[java/:?]
   [junit4]   2> 	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176) ~[metrics-core-3.2.6.jar:3.2.6]
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) ~[java/:?]
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_191]
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_191]
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_191]
   [junit4]   2> 371062 WARN  (updateExecutor-2375-thread-11-processing-x:deleteReplicaOnIndexing_shard1_replica_n1 https:////127.0.0.1:36678//solr//deleteReplicaOnIndexing_shard1_replica_n2 r:core_node3 n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.c.s.i.ConcurrentUpdateSolrClient Failed to parse error response from https://127.0.0.1:36678/solr/deleteReplicaOnIndexing_shard1_replica_n2 due to: java.lang.RuntimeException: Invalid version (expected 2, but 60) or the data in not in 'javabin' format
   [junit4]   2> 371062 ERROR (updateExecutor-2375-thread-11-processing-x:deleteReplicaOnIndexing_shard1_replica_n1 https:////127.0.0.1:36678//solr//deleteReplicaOnIndexing_shard1_replica_n2 r:core_node3 n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.u.ErrorReportingConcurrentUpdateSolrClient error
   [junit4]   2> org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:36678/solr/deleteReplicaOnIndexing_shard1_replica_n2: Can not find: /solr/deleteReplicaOnIndexing_shard1_replica_n2/update
   [junit4]   2> 
   [junit4]   2> 
   [junit4]   2> 
   [junit4]   2> request: https://127.0.0.1:36678/solr/deleteReplicaOnIndexing_shard1_replica_n2/update?update.distrib=FROMLEADER&distrib.from=https%3A%2F%2F127.0.0.1%3A42891%2Fsolr%2FdeleteReplicaOnIndexing_shard1_replica_n1%2F&wt=javabin&version=2
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:385) ~[java/:?]
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:183) ~[java/:?]
   [junit4]   2> 	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176) ~[metrics-core-3.2.6.jar:3.2.6]
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) ~[java/:?]
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_191]
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_191]
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_191]
   [junit4]   2> 371066 WARN  (updateExecutor-2375-thread-1-processing-x:deleteReplicaOnIndexing_shard1_replica_n1 https:////127.0.0.1:36678//solr//deleteReplicaOnIndexing_shard1_replica_n2 r:core_node3 n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.c.s.i.ConcurrentUpdateSolrClient Failed to parse error response from https://127.0.0.1:36678/solr/deleteReplicaOnIndexing_shard1_replica_n2 due to: java.lang.RuntimeException: Invalid version (expected 2, but 60) or the data in not in 'javabin' format
   [junit4]   2> 371067 ERROR (updateExecutor-2375-thread-1-processing-x:deleteReplicaOnIndexing_shard1_replica_n1 https:////127.0.0.1:36678//solr//deleteReplicaOnIndexing_shard1_replica_n2 r:core_node3 n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.u.ErrorReportingConcurrentUpdateSolrClient error
   [junit4]   2> org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:36678/solr/deleteReplicaOnIndexing_shard1_replica_n2: Can not find: /solr/deleteReplicaOnIndexing_shard1_replica_n2/update
   [junit4]   2> 
   [junit4]   2> 
   [junit4]   2> 
   [junit4]   2> request: https://127.0.0.1:36678/solr/deleteReplicaOnIndexing_shard1_replica_n2/update?update.distrib=FROMLEADER&distrib.from=https%3A%2F%2F127.0.0.1%3A42891%2Fsolr%2FdeleteReplicaOnIndexing_shard1_replica_n1%2F&wt=javabin&version=2
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:385) ~[java/:?]
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:183) ~[java/:?]
   [junit4]   2> 	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176) ~[metrics-core-3.2.6.jar:3.2.6]
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) ~[java/:?]
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_191]
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_191]
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_191]
   [junit4]   2> 371071 WARN  (updateExecutor-2375-thread-7-processing-x:deleteReplicaOnIndexing_shard1_replica_n1 https:////127.0.0.1:36678//solr//deleteReplicaOnIndexing_shard1_replica_n2 r:core_node3 n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1) [n:127.0.0.1:42891_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.c.s.i.ConcurrentUpdateSolrClient Failed to parse error response from https://127.0.0.1:36678/solr/deleteReplicaOnIndexing_shard1_replica_n2 due to: java.lang.RuntimeEx

[...truncated too long message...]

ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/null1138040512
     [copy] Copying 240 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/null1138040512
   [delete] Deleting directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/null1138040512

check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] 	confs: [default]
[ivy:cachepath] 	found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] 	found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] 	found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] 	found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] 	found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] 	found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] 	found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] 	found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 31ms :: artifacts dl 2ms
	---------------------------------------------------------------------
	|                  |            modules            ||   artifacts   |
	|       conf       | number| search|dwnlded|evicted|| number|dwnlded|
	---------------------------------------------------------------------
	|      default     |   8   |   0   |   0   |   0   ||   8   |   0   |
	---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...

-jenkins-base:

BUILD SUCCESSFUL
Total time: 101 minutes 2 seconds
Archiving artifacts
java.lang.InterruptedException: no matches found within 10000
	at hudson.FilePath$34.hasMatch(FilePath.java:2678)
	at hudson.FilePath$34.invoke(FilePath.java:2557)
	at hudson.FilePath$34.invoke(FilePath.java:2547)
	at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2918)
Also:   hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
		at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
		at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
		at hudson.remoting.Channel.call(Channel.java:955)
		at hudson.FilePath.act(FilePath.java:1036)
		at hudson.FilePath.act(FilePath.java:1025)
		at hudson.FilePath.validateAntFileMask(FilePath.java:2547)
		at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
		at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
		at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
		at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
		at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
		at hudson.model.Build$BuildExecution.post2(Build.java:186)
		at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
		at hudson.model.Run.execute(Run.java:1819)
		at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
		at hudson.model.ResourceController.execute(ResourceController.java:97)
		at hudson.model.Executor.run(Executor.java:429)
Caused: hudson.FilePath$TunneledInterruptedException
	at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2920)
	at hudson.remoting.UserRequest.perform(UserRequest.java:212)
	at hudson.remoting.UserRequest.perform(UserRequest.java:54)
	at hudson.remoting.Request$2.run(Request.java:369)
	at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:748)
Caused: java.lang.InterruptedException: java.lang.InterruptedException: no matches found within 10000
	at hudson.FilePath.act(FilePath.java:1038)
	at hudson.FilePath.act(FilePath.java:1025)
	at hudson.FilePath.validateAntFileMask(FilePath.java:2547)
	at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
	at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
	at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
	at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
	at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
	at hudson.model.Build$BuildExecution.post2(Build.java:186)
	at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
	at hudson.model.Run.execute(Run.java:1819)
	at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
	at hudson.model.ResourceController.execute(ResourceController.java:97)
	at hudson.model.Executor.run(Executor.java:429)
No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)

[JENKINS] Lucene-Solr-Tests-master - Build # 3055 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-master/3055/

No tests ran.

Build Log:
[...truncated 6 lines...]
ERROR: Error fetching remote repo 'origin'
hudson.plugins.git.GitException: Failed to fetch from git://git.apache.org/lucene-solr.git
	at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:888)
	at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1155)
	at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1186)
	at hudson.scm.SCM.checkout(SCM.java:504)
	at hudson.model.AbstractProject.checkout(AbstractProject.java:1208)
	at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
	at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
	at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
	at hudson.model.Run.execute(Run.java:1794)
	at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
	at hudson.model.ResourceController.execute(ResourceController.java:97)
	at hudson.model.Executor.run(Executor.java:429)
Caused by: hudson.plugins.git.GitException: Command "git config remote.origin.url git://git.apache.org/lucene-solr.git" returned status code 4:
stdout: 
stderr: error: failed to write new configuration file .git/config.lock

	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:2002)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1970)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1966)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommand(CliGitAPIImpl.java:1597)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommand(CliGitAPIImpl.java:1609)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.setRemoteUrl(CliGitAPIImpl.java:1243)
	at hudson.plugins.git.GitAPI.setRemoteUrl(GitAPI.java:160)
	at sun.reflect.GeneratedMethodAccessor550.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at hudson.remoting.RemoteInvocationHandler$RPCRequest.perform(RemoteInvocationHandler.java:929)
	at hudson.remoting.RemoteInvocationHandler$RPCRequest.call(RemoteInvocationHandler.java:903)
	at hudson.remoting.RemoteInvocationHandler$RPCRequest.call(RemoteInvocationHandler.java:855)
	at hudson.remoting.UserRequest.perform(UserRequest.java:212)
	at hudson.remoting.UserRequest.perform(UserRequest.java:54)
	at hudson.remoting.Request$2.run(Request.java:369)
	at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:748)
	Suppressed: hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
		at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
		at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
		at hudson.remoting.Channel.call(Channel.java:955)
		at hudson.remoting.RemoteInvocationHandler.invoke(RemoteInvocationHandler.java:283)
		at com.sun.proxy.$Proxy117.setRemoteUrl(Unknown Source)
		at org.jenkinsci.plugins.gitclient.RemoteGitImpl.setRemoteUrl(RemoteGitImpl.java:295)
		at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:876)
		at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1155)
		at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1186)
		at hudson.scm.SCM.checkout(SCM.java:504)
		at hudson.model.AbstractProject.checkout(AbstractProject.java:1208)
		at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
		at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
		at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
		at hudson.model.Run.execute(Run.java:1794)
		at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
		at hudson.model.ResourceController.execute(ResourceController.java:97)
		at hudson.model.Executor.run(Executor.java:429)
ERROR: Error fetching remote repo 'origin'
Retrying after 10 seconds
 > git rev-parse --is-inside-work-tree # timeout=10
Fetching changes from the remote Git repository
 > git config remote.origin.url git://git.apache.org/lucene-solr.git # timeout=10
ERROR: Error fetching remote repo 'origin'
hudson.plugins.git.GitException: Failed to fetch from git://git.apache.org/lucene-solr.git
	at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:888)
	at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1155)
	at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1186)
	at hudson.scm.SCM.checkout(SCM.java:504)
	at hudson.model.AbstractProject.checkout(AbstractProject.java:1208)
	at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
	at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
	at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
	at hudson.model.Run.execute(Run.java:1794)
	at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
	at hudson.model.ResourceController.execute(ResourceController.java:97)
	at hudson.model.Executor.run(Executor.java:429)
Caused by: hudson.plugins.git.GitException: Command "git config remote.origin.url git://git.apache.org/lucene-solr.git" returned status code 4:
stdout: 
stderr: error: failed to write new configuration file .git/config.lock

	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:2002)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1970)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1966)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommand(CliGitAPIImpl.java:1597)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommand(CliGitAPIImpl.java:1609)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.setRemoteUrl(CliGitAPIImpl.java:1243)
	at hudson.plugins.git.GitAPI.setRemoteUrl(GitAPI.java:160)
	at sun.reflect.GeneratedMethodAccessor550.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at hudson.remoting.RemoteInvocationHandler$RPCRequest.perform(RemoteInvocationHandler.java:929)
	at hudson.remoting.RemoteInvocationHandler$RPCRequest.call(RemoteInvocationHandler.java:903)
	at hudson.remoting.RemoteInvocationHandler$RPCRequest.call(RemoteInvocationHandler.java:855)
	at hudson.remoting.UserRequest.perform(UserRequest.java:212)
	at hudson.remoting.UserRequest.perform(UserRequest.java:54)
	at hudson.remoting.Request$2.run(Request.java:369)
	at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:748)
	Suppressed: hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
		at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
		at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
		at hudson.remoting.Channel.call(Channel.java:955)
		at hudson.remoting.RemoteInvocationHandler.invoke(RemoteInvocationHandler.java:283)
		at com.sun.proxy.$Proxy117.setRemoteUrl(Unknown Source)
		at org.jenkinsci.plugins.gitclient.RemoteGitImpl.setRemoteUrl(RemoteGitImpl.java:295)
		at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:876)
		at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1155)
		at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1186)
		at hudson.scm.SCM.checkout(SCM.java:504)
		at hudson.model.AbstractProject.checkout(AbstractProject.java:1208)
		at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
		at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
		at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
		at hudson.model.Run.execute(Run.java:1794)
		at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
		at hudson.model.ResourceController.execute(ResourceController.java:97)
		at hudson.model.Executor.run(Executor.java:429)
ERROR: Error fetching remote repo 'origin'
Retrying after 10 seconds
 > git rev-parse --is-inside-work-tree # timeout=10
Fetching changes from the remote Git repository
 > git config remote.origin.url git://git.apache.org/lucene-solr.git # timeout=10
ERROR: Error fetching remote repo 'origin'
hudson.plugins.git.GitException: Failed to fetch from git://git.apache.org/lucene-solr.git
	at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:888)
	at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1155)
	at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1186)
	at hudson.scm.SCM.checkout(SCM.java:504)
	at hudson.model.AbstractProject.checkout(AbstractProject.java:1208)
	at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
	at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
	at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
	at hudson.model.Run.execute(Run.java:1794)
	at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
	at hudson.model.ResourceController.execute(ResourceController.java:97)
	at hudson.model.Executor.run(Executor.java:429)
Caused by: hudson.plugins.git.GitException: Command "git config remote.origin.url git://git.apache.org/lucene-solr.git" returned status code 4:
stdout: 
stderr: error: failed to write new configuration file .git/config.lock

	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:2002)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1970)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1966)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommand(CliGitAPIImpl.java:1597)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommand(CliGitAPIImpl.java:1609)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.setRemoteUrl(CliGitAPIImpl.java:1243)
	at hudson.plugins.git.GitAPI.setRemoteUrl(GitAPI.java:160)
	at sun.reflect.GeneratedMethodAccessor550.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at hudson.remoting.RemoteInvocationHandler$RPCRequest.perform(RemoteInvocationHandler.java:929)
	at hudson.remoting.RemoteInvocationHandler$RPCRequest.call(RemoteInvocationHandler.java:903)
	at hudson.remoting.RemoteInvocationHandler$RPCRequest.call(RemoteInvocationHandler.java:855)
	at hudson.remoting.UserRequest.perform(UserRequest.java:212)
	at hudson.remoting.UserRequest.perform(UserRequest.java:54)
	at hudson.remoting.Request$2.run(Request.java:369)
	at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:748)
	Suppressed: hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
		at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
		at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
		at hudson.remoting.Channel.call(Channel.java:955)
		at hudson.remoting.RemoteInvocationHandler.invoke(RemoteInvocationHandler.java:283)
		at com.sun.proxy.$Proxy117.setRemoteUrl(Unknown Source)
		at org.jenkinsci.plugins.gitclient.RemoteGitImpl.setRemoteUrl(RemoteGitImpl.java:295)
		at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:876)
		at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1155)
		at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1186)
		at hudson.scm.SCM.checkout(SCM.java:504)
		at hudson.model.AbstractProject.checkout(AbstractProject.java:1208)
		at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
		at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
		at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
		at hudson.model.Run.execute(Run.java:1794)
		at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
		at hudson.model.ResourceController.execute(ResourceController.java:97)
		at hudson.model.Executor.run(Executor.java:429)
ERROR: Error fetching remote repo 'origin'
Archiving artifacts
Recording test results
ERROR: Step ‘Publish JUnit test result report’ failed: Test reports were found but none of them are new. Did leafNodes run? 
For example, /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/build/analysis/common/test/TEST-org.apache.lucene.analysis.ar.TestArabicAnalyzer.xml is 1 day 23 hr old

Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any

[JENKINS] Lucene-Solr-Tests-master - Build # 3054 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-master/3054/

No tests ran.

Build Log:
[...truncated 6 lines...]
ERROR: Error fetching remote repo 'origin'
hudson.plugins.git.GitException: Failed to fetch from git://git.apache.org/lucene-solr.git
	at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:888)
	at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1155)
	at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1186)
	at hudson.scm.SCM.checkout(SCM.java:504)
	at hudson.model.AbstractProject.checkout(AbstractProject.java:1208)
	at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
	at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
	at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
	at hudson.model.Run.execute(Run.java:1794)
	at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
	at hudson.model.ResourceController.execute(ResourceController.java:97)
	at hudson.model.Executor.run(Executor.java:429)
Caused by: hudson.plugins.git.GitException: Command "git config remote.origin.url git://git.apache.org/lucene-solr.git" returned status code 4:
stdout: 
stderr: error: failed to write new configuration file .git/config.lock

	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:2002)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1970)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1966)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommand(CliGitAPIImpl.java:1597)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommand(CliGitAPIImpl.java:1609)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.setRemoteUrl(CliGitAPIImpl.java:1243)
	at hudson.plugins.git.GitAPI.setRemoteUrl(GitAPI.java:160)
	at sun.reflect.GeneratedMethodAccessor550.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at hudson.remoting.RemoteInvocationHandler$RPCRequest.perform(RemoteInvocationHandler.java:929)
	at hudson.remoting.RemoteInvocationHandler$RPCRequest.call(RemoteInvocationHandler.java:903)
	at hudson.remoting.RemoteInvocationHandler$RPCRequest.call(RemoteInvocationHandler.java:855)
	at hudson.remoting.UserRequest.perform(UserRequest.java:212)
	at hudson.remoting.UserRequest.perform(UserRequest.java:54)
	at hudson.remoting.Request$2.run(Request.java:369)
	at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:748)
	Suppressed: hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
		at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
		at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
		at hudson.remoting.Channel.call(Channel.java:955)
		at hudson.remoting.RemoteInvocationHandler.invoke(RemoteInvocationHandler.java:283)
		at com.sun.proxy.$Proxy117.setRemoteUrl(Unknown Source)
		at org.jenkinsci.plugins.gitclient.RemoteGitImpl.setRemoteUrl(RemoteGitImpl.java:295)
		at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:876)
		at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1155)
		at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1186)
		at hudson.scm.SCM.checkout(SCM.java:504)
		at hudson.model.AbstractProject.checkout(AbstractProject.java:1208)
		at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
		at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
		at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
		at hudson.model.Run.execute(Run.java:1794)
		at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
		at hudson.model.ResourceController.execute(ResourceController.java:97)
		at hudson.model.Executor.run(Executor.java:429)
ERROR: Error fetching remote repo 'origin'
Retrying after 10 seconds
 > git rev-parse --is-inside-work-tree # timeout=10
Fetching changes from the remote Git repository
 > git config remote.origin.url git://git.apache.org/lucene-solr.git # timeout=10
ERROR: Error fetching remote repo 'origin'
hudson.plugins.git.GitException: Failed to fetch from git://git.apache.org/lucene-solr.git
	at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:888)
	at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1155)
	at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1186)
	at hudson.scm.SCM.checkout(SCM.java:504)
	at hudson.model.AbstractProject.checkout(AbstractProject.java:1208)
	at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
	at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
	at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
	at hudson.model.Run.execute(Run.java:1794)
	at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
	at hudson.model.ResourceController.execute(ResourceController.java:97)
	at hudson.model.Executor.run(Executor.java:429)
Caused by: hudson.plugins.git.GitException: Command "git config remote.origin.url git://git.apache.org/lucene-solr.git" returned status code 4:
stdout: 
stderr: error: failed to write new configuration file .git/config.lock

	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:2002)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1970)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1966)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommand(CliGitAPIImpl.java:1597)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommand(CliGitAPIImpl.java:1609)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.setRemoteUrl(CliGitAPIImpl.java:1243)
	at hudson.plugins.git.GitAPI.setRemoteUrl(GitAPI.java:160)
	at sun.reflect.GeneratedMethodAccessor550.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at hudson.remoting.RemoteInvocationHandler$RPCRequest.perform(RemoteInvocationHandler.java:929)
	at hudson.remoting.RemoteInvocationHandler$RPCRequest.call(RemoteInvocationHandler.java:903)
	at hudson.remoting.RemoteInvocationHandler$RPCRequest.call(RemoteInvocationHandler.java:855)
	at hudson.remoting.UserRequest.perform(UserRequest.java:212)
	at hudson.remoting.UserRequest.perform(UserRequest.java:54)
	at hudson.remoting.Request$2.run(Request.java:369)
	at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:748)
	Suppressed: hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
		at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
		at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
		at hudson.remoting.Channel.call(Channel.java:955)
		at hudson.remoting.RemoteInvocationHandler.invoke(RemoteInvocationHandler.java:283)
		at com.sun.proxy.$Proxy117.setRemoteUrl(Unknown Source)
		at org.jenkinsci.plugins.gitclient.RemoteGitImpl.setRemoteUrl(RemoteGitImpl.java:295)
		at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:876)
		at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1155)
		at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1186)
		at hudson.scm.SCM.checkout(SCM.java:504)
		at hudson.model.AbstractProject.checkout(AbstractProject.java:1208)
		at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
		at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
		at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
		at hudson.model.Run.execute(Run.java:1794)
		at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
		at hudson.model.ResourceController.execute(ResourceController.java:97)
		at hudson.model.Executor.run(Executor.java:429)
ERROR: Error fetching remote repo 'origin'
Retrying after 10 seconds
 > git rev-parse --is-inside-work-tree # timeout=10
Fetching changes from the remote Git repository
 > git config remote.origin.url git://git.apache.org/lucene-solr.git # timeout=10
ERROR: Error fetching remote repo 'origin'
hudson.plugins.git.GitException: Failed to fetch from git://git.apache.org/lucene-solr.git
	at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:888)
	at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1155)
	at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1186)
	at hudson.scm.SCM.checkout(SCM.java:504)
	at hudson.model.AbstractProject.checkout(AbstractProject.java:1208)
	at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
	at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
	at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
	at hudson.model.Run.execute(Run.java:1794)
	at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
	at hudson.model.ResourceController.execute(ResourceController.java:97)
	at hudson.model.Executor.run(Executor.java:429)
Caused by: hudson.plugins.git.GitException: Command "git config remote.origin.url git://git.apache.org/lucene-solr.git" returned status code 4:
stdout: 
stderr: error: failed to write new configuration file .git/config.lock

	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:2002)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1970)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1966)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommand(CliGitAPIImpl.java:1597)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommand(CliGitAPIImpl.java:1609)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.setRemoteUrl(CliGitAPIImpl.java:1243)
	at hudson.plugins.git.GitAPI.setRemoteUrl(GitAPI.java:160)
	at sun.reflect.GeneratedMethodAccessor550.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at hudson.remoting.RemoteInvocationHandler$RPCRequest.perform(RemoteInvocationHandler.java:929)
	at hudson.remoting.RemoteInvocationHandler$RPCRequest.call(RemoteInvocationHandler.java:903)
	at hudson.remoting.RemoteInvocationHandler$RPCRequest.call(RemoteInvocationHandler.java:855)
	at hudson.remoting.UserRequest.perform(UserRequest.java:212)
	at hudson.remoting.UserRequest.perform(UserRequest.java:54)
	at hudson.remoting.Request$2.run(Request.java:369)
	at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:748)
	Suppressed: hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
		at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
		at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
		at hudson.remoting.Channel.call(Channel.java:955)
		at hudson.remoting.RemoteInvocationHandler.invoke(RemoteInvocationHandler.java:283)
		at com.sun.proxy.$Proxy117.setRemoteUrl(Unknown Source)
		at org.jenkinsci.plugins.gitclient.RemoteGitImpl.setRemoteUrl(RemoteGitImpl.java:295)
		at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:876)
		at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1155)
		at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1186)
		at hudson.scm.SCM.checkout(SCM.java:504)
		at hudson.model.AbstractProject.checkout(AbstractProject.java:1208)
		at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
		at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
		at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
		at hudson.model.Run.execute(Run.java:1794)
		at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
		at hudson.model.ResourceController.execute(ResourceController.java:97)
		at hudson.model.Executor.run(Executor.java:429)
ERROR: Error fetching remote repo 'origin'
Archiving artifacts
Recording test results
ERROR: Step ‘Publish JUnit test result report’ failed: Test reports were found but none of them are new. Did leafNodes run? 
For example, /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/build/analysis/common/test/TEST-org.apache.lucene.analysis.ar.TestArabicAnalyzer.xml is 1 day 22 hr old

Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any

[JENKINS] Lucene-Solr-Tests-master - Build # 3053 - Failure

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-master/3053/

No tests ran.

Build Log:
[...truncated 18 lines...]
ERROR: Error fetching remote repo 'origin'
hudson.plugins.git.GitException: Failed to fetch from git://git.apache.org/lucene-solr.git
	at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:888)
	at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1155)
	at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1186)
	at hudson.scm.SCM.checkout(SCM.java:504)
	at hudson.model.AbstractProject.checkout(AbstractProject.java:1208)
	at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
	at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
	at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
	at hudson.model.Run.execute(Run.java:1794)
	at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
	at hudson.model.ResourceController.execute(ResourceController.java:97)
	at hudson.model.Executor.run(Executor.java:429)
Caused by: hudson.plugins.git.GitException: Command "git config remote.origin.url git://git.apache.org/lucene-solr.git" returned status code 4:
stdout: 
stderr: error: failed to write new configuration file .git/config.lock

	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:2002)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1970)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1966)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommand(CliGitAPIImpl.java:1597)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommand(CliGitAPIImpl.java:1609)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.setRemoteUrl(CliGitAPIImpl.java:1243)
	at hudson.plugins.git.GitAPI.setRemoteUrl(GitAPI.java:160)
	at sun.reflect.GeneratedMethodAccessor550.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at hudson.remoting.RemoteInvocationHandler$RPCRequest.perform(RemoteInvocationHandler.java:929)
	at hudson.remoting.RemoteInvocationHandler$RPCRequest.call(RemoteInvocationHandler.java:903)
	at hudson.remoting.RemoteInvocationHandler$RPCRequest.call(RemoteInvocationHandler.java:855)
	at hudson.remoting.UserRequest.perform(UserRequest.java:212)
	at hudson.remoting.UserRequest.perform(UserRequest.java:54)
	at hudson.remoting.Request$2.run(Request.java:369)
	at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:748)
	Suppressed: hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
		at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
		at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
		at hudson.remoting.Channel.call(Channel.java:955)
		at hudson.remoting.RemoteInvocationHandler.invoke(RemoteInvocationHandler.java:283)
		at com.sun.proxy.$Proxy117.setRemoteUrl(Unknown Source)
		at org.jenkinsci.plugins.gitclient.RemoteGitImpl.setRemoteUrl(RemoteGitImpl.java:295)
		at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:876)
		at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1155)
		at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1186)
		at hudson.scm.SCM.checkout(SCM.java:504)
		at hudson.model.AbstractProject.checkout(AbstractProject.java:1208)
		at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
		at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
		at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
		at hudson.model.Run.execute(Run.java:1794)
		at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
		at hudson.model.ResourceController.execute(ResourceController.java:97)
		at hudson.model.Executor.run(Executor.java:429)
ERROR: Error fetching remote repo 'origin'
Retrying after 10 seconds
 > git rev-parse --is-inside-work-tree # timeout=10
Fetching changes from the remote Git repository
 > git config remote.origin.url git://git.apache.org/lucene-solr.git # timeout=10
ERROR: Error fetching remote repo 'origin'
hudson.plugins.git.GitException: Failed to fetch from git://git.apache.org/lucene-solr.git
	at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:888)
	at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1155)
	at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1186)
	at hudson.scm.SCM.checkout(SCM.java:504)
	at hudson.model.AbstractProject.checkout(AbstractProject.java:1208)
	at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
	at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
	at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
	at hudson.model.Run.execute(Run.java:1794)
	at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
	at hudson.model.ResourceController.execute(ResourceController.java:97)
	at hudson.model.Executor.run(Executor.java:429)
Caused by: hudson.plugins.git.GitException: Command "git config remote.origin.url git://git.apache.org/lucene-solr.git" returned status code 4:
stdout: 
stderr: error: failed to write new configuration file .git/config.lock

	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:2002)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1970)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1966)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommand(CliGitAPIImpl.java:1597)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommand(CliGitAPIImpl.java:1609)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.setRemoteUrl(CliGitAPIImpl.java:1243)
	at hudson.plugins.git.GitAPI.setRemoteUrl(GitAPI.java:160)
	at sun.reflect.GeneratedMethodAccessor550.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at hudson.remoting.RemoteInvocationHandler$RPCRequest.perform(RemoteInvocationHandler.java:929)
	at hudson.remoting.RemoteInvocationHandler$RPCRequest.call(RemoteInvocationHandler.java:903)
	at hudson.remoting.RemoteInvocationHandler$RPCRequest.call(RemoteInvocationHandler.java:855)
	at hudson.remoting.UserRequest.perform(UserRequest.java:212)
	at hudson.remoting.UserRequest.perform(UserRequest.java:54)
	at hudson.remoting.Request$2.run(Request.java:369)
	at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:748)
	Suppressed: hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
		at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
		at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
		at hudson.remoting.Channel.call(Channel.java:955)
		at hudson.remoting.RemoteInvocationHandler.invoke(RemoteInvocationHandler.java:283)
		at com.sun.proxy.$Proxy117.setRemoteUrl(Unknown Source)
		at org.jenkinsci.plugins.gitclient.RemoteGitImpl.setRemoteUrl(RemoteGitImpl.java:295)
		at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:876)
		at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1155)
		at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1186)
		at hudson.scm.SCM.checkout(SCM.java:504)
		at hudson.model.AbstractProject.checkout(AbstractProject.java:1208)
		at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
		at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
		at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
		at hudson.model.Run.execute(Run.java:1794)
		at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
		at hudson.model.ResourceController.execute(ResourceController.java:97)
		at hudson.model.Executor.run(Executor.java:429)
ERROR: Error fetching remote repo 'origin'
Retrying after 10 seconds
 > git rev-parse --is-inside-work-tree # timeout=10
Fetching changes from the remote Git repository
 > git config remote.origin.url git://git.apache.org/lucene-solr.git # timeout=10
ERROR: Error fetching remote repo 'origin'
hudson.plugins.git.GitException: Failed to fetch from git://git.apache.org/lucene-solr.git
	at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:888)
	at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1155)
	at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1186)
	at hudson.scm.SCM.checkout(SCM.java:504)
	at hudson.model.AbstractProject.checkout(AbstractProject.java:1208)
	at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
	at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
	at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
	at hudson.model.Run.execute(Run.java:1794)
	at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
	at hudson.model.ResourceController.execute(ResourceController.java:97)
	at hudson.model.Executor.run(Executor.java:429)
Caused by: hudson.plugins.git.GitException: Command "git config remote.origin.url git://git.apache.org/lucene-solr.git" returned status code 4:
stdout: 
stderr: error: failed to write new configuration file .git/config.lock

	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:2002)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1970)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1966)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommand(CliGitAPIImpl.java:1597)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommand(CliGitAPIImpl.java:1609)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.setRemoteUrl(CliGitAPIImpl.java:1243)
	at hudson.plugins.git.GitAPI.setRemoteUrl(GitAPI.java:160)
	at sun.reflect.GeneratedMethodAccessor550.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at hudson.remoting.RemoteInvocationHandler$RPCRequest.perform(RemoteInvocationHandler.java:929)
	at hudson.remoting.RemoteInvocationHandler$RPCRequest.call(RemoteInvocationHandler.java:903)
	at hudson.remoting.RemoteInvocationHandler$RPCRequest.call(RemoteInvocationHandler.java:855)
	at hudson.remoting.UserRequest.perform(UserRequest.java:212)
	at hudson.remoting.UserRequest.perform(UserRequest.java:54)
	at hudson.remoting.Request$2.run(Request.java:369)
	at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:748)
	Suppressed: hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
		at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
		at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
		at hudson.remoting.Channel.call(Channel.java:955)
		at hudson.remoting.RemoteInvocationHandler.invoke(RemoteInvocationHandler.java:283)
		at com.sun.proxy.$Proxy117.setRemoteUrl(Unknown Source)
		at org.jenkinsci.plugins.gitclient.RemoteGitImpl.setRemoteUrl(RemoteGitImpl.java:295)
		at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:876)
		at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1155)
		at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1186)
		at hudson.scm.SCM.checkout(SCM.java:504)
		at hudson.model.AbstractProject.checkout(AbstractProject.java:1208)
		at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
		at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
		at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
		at hudson.model.Run.execute(Run.java:1794)
		at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
		at hudson.model.ResourceController.execute(ResourceController.java:97)
		at hudson.model.Executor.run(Executor.java:429)
ERROR: Error fetching remote repo 'origin'
Archiving artifacts
Recording test results
ERROR: Step ‘Publish JUnit test result report’ failed: Test reports were found but none of them are new. Did leafNodes run? 
For example, /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/build/analysis/common/test/TEST-org.apache.lucene.analysis.ar.TestArabicAnalyzer.xml is 1 day 21 hr old

Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any

[JENKINS] Lucene-Solr-Tests-master - Build # 3052 - Still Unstable

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-master/3052/

1 tests failed.
FAILED:  org.apache.solr.cloud.autoscaling.sim.TestSimLargeCluster.testAddNode

Error Message:
unexpected number of MOVEREPLICA ops

Stack Trace:
java.lang.AssertionError: unexpected number of MOVEREPLICA ops
	at __randomizedtesting.SeedInfo.seed([AE28E86C45BF8A88:9C7F5CF8AF20590]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.cloud.autoscaling.sim.TestSimLargeCluster.testAddNode(TestSimLargeCluster.java:381)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:935)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:971)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:985)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:944)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:830)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 14704 lines...]
   [junit4] Suite: org.apache.solr.cloud.autoscaling.sim.TestSimLargeCluster
   [junit4]   2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.sim.TestSimLargeCluster_AE28E86C45BF8A88-001/init-core-data-001
   [junit4]   2> 3231707 INFO  (SUITE-TestSimLargeCluster-seed#[AE28E86C45BF8A88]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 3231709 INFO  (SUITE-TestSimLargeCluster-seed#[AE28E86C45BF8A88]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 3231709 INFO  (SUITE-TestSimLargeCluster-seed#[AE28E86C45BF8A88]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 3231738 DEBUG (SUITE-TestSimLargeCluster-seed#[AE28E86C45BF8A88]-worker) [    ] o.a.s.c.a.s.SimClusterStateProvider --- new Overseer leader: 127.0.0.1:10006_solr
   [junit4]   2> 3231739 DEBUG (SUITE-TestSimLargeCluster-seed#[AE28E86C45BF8A88]-worker) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=0
   [junit4]   2> 3231739 DEBUG (SUITE-TestSimLargeCluster-seed#[AE28E86C45BF8A88]-worker) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 0
   [junit4]   2> 3231739 INFO  (SUITE-TestSimLargeCluster-seed#[AE28E86C45BF8A88]-worker) [    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 3231743 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
   [junit4]   2> 3231743 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 0
   [junit4]   2> 3231743 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion -1
   [junit4]   2> 3231743 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 0
   [junit4]   2> 3231763 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10082_solr, 127.0.0.1:10018_solr, 127.0.0.1:10049_solr, 127.0.0.1:10104_solr, 127.0.0.1:10020_solr, 127.0.0.1:10065_solr, 127.0.0.1:10021_solr, 127.0.0.1:10066_solr, 127.0.0.1:10035_solr, 127.0.0.1:10051_solr, 127.0.0.1:10019_solr, 127.0.0.1:10096_solr, 127.0.0.1:10022_solr, 127.0.0.1:10033_solr, 127.0.0.1:10105_solr, 127.0.0.1:10017_solr, 127.0.0.1:10047_solr, 127.0.0.1:10102_solr, 127.0.0.1:10095_solr, 127.0.0.1:10081_solr, 127.0.0.1:10084_solr, 127.0.0.1:10036_solr, 127.0.0.1:10052_solr, 127.0.0.1:10006_solr, 127.0.0.1:10079_solr, 127.0.0.1:10098_solr, 127.0.0.1:10054_solr, 127.0.0.1:10099_solr, 127.0.0.1:10101_solr, 127.0.0.1:10071_solr, 127.0.0.1:10023_solr, 127.0.0.1:10076_solr, 127.0.0.1:10007_solr, 127.0.0.1:10055_solr, 127.0.0.1:10010_solr, 127.0.0.1:10016_solr, 127.0.0.1:10070_solr, 127.0.0.1:10077_solr, 127.0.0.1:10038_solr, 127.0.0.1:10032_solr, 127.0.0.1:10093_solr, 127.0.0.1:10061_solr, 127.0.0.1:10067_solr, 127.0.0.1:10086_solr, 127.0.0.1:10089_solr, 127.0.0.1:10045_solr, 127.0.0.1:10042_solr, 127.0.0.1:10029_solr, 127.0.0.1:10083_solr, 127.0.0.1:10080_solr, 127.0.0.1:10048_solr, 127.0.0.1:10064_solr, 127.0.0.1:10043_solr, 127.0.0.1:10057_solr, 127.0.0.1:10074_solr, 127.0.0.1:10026_solr, 127.0.0.1:10012_solr, 127.0.0.1:10060_solr, 127.0.0.1:10073_solr, 127.0.0.1:10058_solr, 127.0.0.1:10013_solr, 127.0.0.1:10090_solr, 127.0.0.1:10088_solr, 127.0.0.1:10027_solr, 127.0.0.1:10039_solr, 127.0.0.1:10025_solr, 127.0.0.1:10044_solr, 127.0.0.1:10028_solr, 127.0.0.1:10092_solr, 127.0.0.1:10041_solr, 127.0.0.1:10087_solr, 127.0.0.1:10009_solr, 127.0.0.1:10030_solr, 127.0.0.1:10014_solr, 127.0.0.1:10011_solr, 127.0.0.1:10040_solr, 127.0.0.1:10085_solr, 127.0.0.1:10046_solr, 127.0.0.1:10068_solr, 127.0.0.1:10063_solr, 127.0.0.1:10015_solr, 127.0.0.1:10069_solr, 127.0.0.1:10062_solr, 127.0.0.1:10008_solr, 127.0.0.1:10024_solr, 127.0.0.1:10050_solr, 127.0.0.1:10075_solr, 127.0.0.1:10078_solr, 127.0.0.1:10097_solr, 127.0.0.1:10031_solr, 127.0.0.1:10034_solr, 127.0.0.1:10091_solr, 127.0.0.1:10094_solr, 127.0.0.1:10100_solr, 127.0.0.1:10037_solr, 127.0.0.1:10053_solr, 127.0.0.1:10059_solr, 127.0.0.1:10103_solr, 127.0.0.1:10056_solr, 127.0.0.1:10072_solr]
   [junit4]   2> 3231787 DEBUG (ScheduledTrigger-8233-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100 and last live nodes: 100
   [junit4]   2> 3231787 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 3231787 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion 0
   [junit4]   2> 3231904 DEBUG (ScheduledTrigger-8233-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100 and last live nodes: 100
   [junit4]   2> 3231917 INFO  (TEST-TestSimLargeCluster.testSearchRate-seed#[AE28E86C45BF8A88]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testSearchRate
   [junit4]   2> 3231917 DEBUG (TEST-TestSimLargeCluster.testSearchRate-seed#[AE28E86C45BF8A88]) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
   [junit4]   2> 3231917 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
   [junit4]   2> 3231917 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
   [junit4]   2> 3231917 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 3231917 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
   [junit4]   2> 3231917 INFO  (TEST-TestSimLargeCluster.testSearchRate-seed#[AE28E86C45BF8A88]) [    ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
   [junit4]   2> 3231918 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 3231918 DEBUG (TEST-TestSimLargeCluster.testSearchRate-seed#[AE28E86C45BF8A88]) [    ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
   [junit4]   2> 3231918 DEBUG (TEST-TestSimLargeCluster.testSearchRate-seed#[AE28E86C45BF8A88]) [    ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now
   [junit4]   2> 3231918 DEBUG (TEST-TestSimLargeCluster.testSearchRate-seed#[AE28E86C45BF8A88]) [    ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
   [junit4]   2> 3231918 DEBUG (TEST-TestSimLargeCluster.testSearchRate-seed#[AE28E86C45BF8A88]) [    ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor
   [junit4]   2> 3231925 DEBUG (TEST-TestSimLargeCluster.testSearchRate-seed#[AE28E86C45BF8A88]) [    ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
   [junit4]   2> 3231925 DEBUG (TEST-TestSimLargeCluster.testSearchRate-seed#[AE28E86C45BF8A88]) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 3231925 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
   [junit4]   2> 3231926 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2
   [junit4]   2> 3231926 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion -1
   [junit4]   2> 3231926 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
   [junit4]   2> 3231941 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10082_solr, 127.0.0.1:10018_solr, 127.0.0.1:10049_solr, 127.0.0.1:10104_solr, 127.0.0.1:10020_solr, 127.0.0.1:10065_solr, 127.0.0.1:10021_solr, 127.0.0.1:10066_solr, 127.0.0.1:10035_solr, 127.0.0.1:10051_solr, 127.0.0.1:10019_solr, 127.0.0.1:10096_solr, 127.0.0.1:10022_solr, 127.0.0.1:10033_solr, 127.0.0.1:10105_solr, 127.0.0.1:10017_solr, 127.0.0.1:10047_solr, 127.0.0.1:10102_solr, 127.0.0.1:10095_solr, 127.0.0.1:10081_solr, 127.0.0.1:10084_solr, 127.0.0.1:10036_solr, 127.0.0.1:10052_solr, 127.0.0.1:10006_solr, 127.0.0.1:10079_solr, 127.0.0.1:10098_solr, 127.0.0.1:10054_solr, 127.0.0.1:10099_solr, 127.0.0.1:10101_solr, 127.0.0.1:10071_solr, 127.0.0.1:10023_solr, 127.0.0.1:10076_solr, 127.0.0.1:10007_solr, 127.0.0.1:10055_solr, 127.0.0.1:10010_solr, 127.0.0.1:10016_solr, 127.0.0.1:10070_solr, 127.0.0.1:10077_solr, 127.0.0.1:10038_solr, 127.0.0.1:10032_solr, 127.0.0.1:10093_solr, 127.0.0.1:10061_solr, 127.0.0.1:10067_solr, 127.0.0.1:10086_solr, 127.0.0.1:10089_solr, 127.0.0.1:10045_solr, 127.0.0.1:10042_solr, 127.0.0.1:10029_solr, 127.0.0.1:10083_solr, 127.0.0.1:10080_solr, 127.0.0.1:10048_solr, 127.0.0.1:10064_solr, 127.0.0.1:10043_solr, 127.0.0.1:10057_solr, 127.0.0.1:10074_solr, 127.0.0.1:10026_solr, 127.0.0.1:10012_solr, 127.0.0.1:10060_solr, 127.0.0.1:10073_solr, 127.0.0.1:10058_solr, 127.0.0.1:10013_solr, 127.0.0.1:10090_solr, 127.0.0.1:10088_solr, 127.0.0.1:10027_solr, 127.0.0.1:10039_solr, 127.0.0.1:10025_solr, 127.0.0.1:10044_solr, 127.0.0.1:10028_solr, 127.0.0.1:10092_solr, 127.0.0.1:10041_solr, 127.0.0.1:10087_solr, 127.0.0.1:10009_solr, 127.0.0.1:10030_solr, 127.0.0.1:10014_solr, 127.0.0.1:10011_solr, 127.0.0.1:10040_solr, 127.0.0.1:10085_solr, 127.0.0.1:10046_solr, 127.0.0.1:10068_solr, 127.0.0.1:10063_solr, 127.0.0.1:10015_solr, 127.0.0.1:10069_solr, 127.0.0.1:10062_solr, 127.0.0.1:10008_solr, 127.0.0.1:10024_solr, 127.0.0.1:10050_solr, 127.0.0.1:10075_solr, 127.0.0.1:10078_solr, 127.0.0.1:10097_solr, 127.0.0.1:10031_solr, 127.0.0.1:10034_solr, 127.0.0.1:10091_solr, 127.0.0.1:10094_solr, 127.0.0.1:10100_solr, 127.0.0.1:10037_solr, 127.0.0.1:10053_solr, 127.0.0.1:10059_solr, 127.0.0.1:10103_solr, 127.0.0.1:10056_solr, 127.0.0.1:10072_solr]
   [junit4]   2> 3231953 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 3231953 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
   [junit4]   2> 3231953 DEBUG (ScheduledTrigger-8238-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100 and last live nodes: 100
   [junit4]   2> 3232052 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100 and last live nodes: 100
   [junit4]   2> 3232072 DEBUG (ScheduledTrigger-8238-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100 and last live nodes: 100
   [junit4]   2> 3232092 DEBUG (ScheduledTrigger-8238-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100 and last live nodes: 100
   [junit4]   2> 3232123 DEBUG (ScheduledTrigger-8238-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100 and last live nodes: 100
   [junit4]   2> 3232126 DEBUG (simCloudManagerPool-8237-thread-1) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=1
   [junit4]   2> 3232126 DEBUG (simCloudManagerPool-8237-thread-1) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 1
   [junit4]   2> 3232140 DEBUG (simCloudManagerPool-8237-thread-1) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 3232141 DEBUG (simCloudManagerPool-8237-thread-1) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 3
   [junit4]   2> 3232164 DEBUG (ScheduledTrigger-8238-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100 and last live nodes: 100
   [junit4]   2> 3232164 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
   [junit4]   2> 3232165 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 3232165 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
   [junit4]   2> 3232168 DEBUG (simCloudManagerPool-8237-thread-2) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 3232168 DEBUG (simCloudManagerPool-8237-thread-2) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 4
   [junit4]   2> 3232169 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 4
   [junit4]   2> 3232169 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
   [junit4]   2> 3232169 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 3232169 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion 4
   [junit4]   2> 3232176 DEBUG (simCloudManagerPool-8237-thread-3) [    ] o.a.s.c.a.s.SimClusterStateProvider -- simCreateCollection testSearchRate, currentVersion=2
   [junit4]   2> 3232176 DEBUG (simCloudManagerPool-8237-thread-3) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=2
   [junit4]   2> 3232176 DEBUG (simCloudManagerPool-8237-thread-3) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 2
   [junit4]   2> 3232187 DEBUG (simCloudManagerPool-8237-thread-3) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=3
   [junit4]   2> 3232196 DEBUG (simCloudManagerPool-8237-thread-3) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 3
   [junit4]   2> 3233177 DEBUG (simCloudManagerPool-8237-thread-4) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=4
   [junit4]   2> 3233181 DEBUG (simCloudManagerPool-8237-thread-4) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 4
   [junit4]   2> 3233189 DEBUG (simCloudManagerPool-8237-thread-3) [    ] o.a.s.c.a.s.SimClusterStateProvider -- finished createCollection testSearchRate, currentVersion=5
   [junit4]   2> 3233221 DEBUG (simCloudManagerPool-8237-thread-10) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=5
   [junit4]   2> 3233222 DEBUG (simCloudManagerPool-8237-thread-10) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 5
   [junit4]   2> 3233232 DEBUG (simCloudManagerPool-8237-thread-6) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=6
   [junit4]   2> 3233254 DEBUG (simCloudManagerPool-8237-thread-6) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 6
   [junit4]   2> 3233264 DEBUG (simCloudManagerPool-8237-thread-5) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=7
   [junit4]   2> 3233265 DEBUG (simCloudManagerPool-8237-thread-5) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 7
   [junit4]   2> 3233275 DEBUG (simCloudManagerPool-8237-thread-7) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=8
   [junit4]   2> 3233276 DEBUG (simCloudManagerPool-8237-thread-7) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 8
   [junit4]   2> 3233286 DEBUG (simCloudManagerPool-8237-thread-8) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=9
   [junit4]   2> 3233286 DEBUG (simCloudManagerPool-8237-thread-8) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 9
   [junit4]   2> 3233297 DEBUG (simCloudManagerPool-8237-thread-12) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=10
   [junit4]   2> 3233297 DEBUG (simCloudManagerPool-8237-thread-12) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 10
   [junit4]   2> 3233297 DEBUG (simCloudManagerPool-8237-thread-11) [    ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testSearchRate / shard1 (currentVersion=11): {"core_node1":{
   [junit4]   2>     "core":"testSearchRate_shard1_replica_n1",
   [junit4]   2>     "shard":"shard1",
   [junit4]   2>     "collection":"testSearchRate",
   [junit4]   2>     "node_name":"127.0.0.1:10072_solr",
   [junit4]   2>     "type":"NRT",
   [junit4]   2>     "leader":"true",
   [junit4]   2>     "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>     "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>     "INDEX.sizeInBytes":10240,
   [junit4]   2>     "state":"active",
   [junit4]   2>     "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>     "SEARCHER.searcher.numDocs":0}}
   [junit4]   2> 3233308 DEBUG (simCloudManagerPool-8237-thread-13) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=11
   [junit4]   2> 3233308 DEBUG (simCloudManagerPool-8237-thread-13) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 11
   [junit4]   2> 3233319 DEBUG (simCloudManagerPool-8237-thread-14) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=12
   [junit4]   2> 3233319 DEBUG (simCloudManagerPool-8237-thread-14) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 12
   [junit4]   2> 3233319 DEBUG (TEST-TestSimLargeCluster.testSearchRate-seed#[AE28E86C45BF8A88]) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=13
   [junit4]   2> 3233319 DEBUG (TEST-TestSimLargeCluster.testSearchRate-seed#[AE28E86C45BF8A88]) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 13
   [junit4]   2> 3233319 INFO  (TEST-TestSimLargeCluster.testSearchRate-seed#[AE28E86C45BF8A88]) [    ] o.a.s.c.CloudTestUtils -- shards without leaders: [shard2]
   [junit4]   2> 3233322 INFO  (TEST-TestSimLargeCluster.testSearchRate-seed#[AE28E86C45BF8A88]) [    ] o.a.s.c.CloudTestUtils -- shards without leaders: [shard2]
   [junit4]   2> 3233324 INFO  (TEST-TestSimLargeCluster.testSearchRate-seed#[AE28E86C45BF8A88]) [    ] o.a.s.c.CloudTestUtils -- shards without leaders: [shard2]
   [junit4]   2> 3233326 INFO  (TEST-TestSimLargeCluster.testSearchRate-seed#[AE28E86C45BF8A88]) [    ] o.a.s.c.CloudTestUtils -- shards without leaders: [shard2]
   [junit4]   2> 3233328 INFO  (TEST-TestSimLargeCluster.testSearchRate-seed#[AE28E86C45BF8A88]) [    ] o.a.s.c.CloudTestUtils -- shards without leaders: [shard2]
   [junit4]   2> 3233330 DEBUG (simCloudManagerPool-8237-thread-15) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=14
   [junit4]   2> 3233330 DEBUG (simCloudManagerPool-8237-thread-15) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 14
   [junit4]   2> 3233330 INFO  (TEST-TestSimLargeCluster.testSearchRate-seed#[AE28E86C45BF8A88]) [    ] o.a.s.c.CloudTestUtils -- shards without leaders: [shard2]
   [junit4]   2> 3233333 INFO  (TEST-TestSimLargeCluster.testSearchRate-seed#[AE28E86C45BF8A88]) [    ] o.a.s.c.CloudTestUtils -- shards without leaders: [shard2]
   [junit4]   2> 3233335 INFO  (TEST-TestSimLargeCluster.testSearchRate-seed#[AE28E86C45BF8A88]) [    ] o.a.s.c.CloudTestUtils -- shards without leaders: [shard2]
   [junit4]   2> 3233337 INFO  (TEST-TestSimLargeCluster.testSearchRate-seed#[AE28E86C45BF8A88]) [    ] o.a.s.c.CloudTestUtils -- shards without leaders: [shard2]
   [junit4]   2> 3233339 INFO  (TEST-TestSimLargeCluster.testSearchRate-seed#[AE28E86C45BF8A88]) [    ] o.a.s.c.CloudTestUtils -- shards without leaders: [shard2]
   [junit4]   2> 3233340 DEBUG (simCloudManagerPool-8237-thread-16) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=15
   [junit4]   2> 3233341 DEBUG (simCloudManagerPool-8237-thread-16) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 15
   [junit4]   2> 3233341 INFO  (TEST-TestSimLargeCluster.testSearchRate-seed#[AE28E86C45BF8A88]) [    ] o.a.s.c.CloudTestUtils -- shards without leaders: [shard2]
   [junit4]   2> 3233344 INFO  (TEST-TestSimLargeCluster.testSearchRate-seed#[AE28E86C45BF8A88]) [    ] o.a.s.c.CloudTestUtils -- shards without leaders: [shard2]
   [junit4]   2> 3233346 INFO  (TEST-TestSimLargeCluster.testSearchRate-seed#[AE28E86C45BF8A88]) [    ] o.a.s.c.CloudTestUtils -- shards without leaders: [shard2]
   [junit4]   2> 3233348 INFO  (TEST-TestSimLargeCluster.testSearchRate-seed#[AE28E86C45BF8A88]) [    ] o.a.s.c.CloudTestUtils -- shards without leaders: [shard2]
   [junit4]   2> 3233350 INFO  (TEST-TestSimLargeCluster.testSearchRate-seed#[AE28E86C45BF8A88]) [    ] o.a.s.c.CloudTestUtils -- shards without leaders: [shard2]
   [junit4]   2> 3233351 DEBUG (simCloudManagerPool-8237-thread-17) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=16
   [junit4]   2> 3233352 DEBUG (simCloudManagerPool-8237-thread-17) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 16
   [junit4]   2> 3233352 INFO  (TEST-TestSimLargeCluster.testSearchRate-seed#[AE28E86C45BF8A88]) [    ] o.a.s.c.CloudTestUtils -- shards without leaders: [shard2]
   [junit4]   2> 3233355 INFO  (TEST-TestSimLargeCluster.testSearchRate-seed#[AE28E86C45BF8A88]) [    ] o.a.s.c.CloudTestUtils -- shards without leaders: [shard2]
   [junit4]   2> 3233357 INFO  (TEST-TestSimLargeCluster.testSearchRate-seed#[AE28E86C45BF8A88]) [    ] o.a.s.c.CloudTestUtils -- shards without leaders: [shard2]
   [junit4]   2> 3233359 INFO  (TEST-TestSimLargeCluster.testSearchRate-seed#[AE28E86C45BF8A88]) [    ] o.a.s.c.CloudTestUtils -- shards without leaders: [shard2]
   [junit4]   2> 3233361 INFO  (TEST-TestSimLargeCluster.testSearchRate-seed#[AE28E86C45BF8A88]) [    ] o.a.s.c.CloudTestUtils -- shards without leaders: [shard2]
   [junit4]   2> 3233362 DEBUG (simCloudManagerPool-8237-thread-18) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=17
   [junit4]   2> 3233363 DEBUG (simCloudManagerPool-8237-thread-18) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 17
   [junit4]   2> 3233364 INFO  (TEST-TestSimLargeCluster.testSearchRate-seed#[AE28E86C45BF8A88]) [    ] o.a.s.c.CloudTestUtils -- shards without leaders: [shard2]
   [junit4]   2> 3233366 INFO  (TEST-TestSimLargeCluster.testSearchRate-seed#[AE28E86C45BF8A88]) [    ] o.a.s.c.CloudTestUtils -- shards without leaders: [shard2]
   [junit4]   2> 3233379 INFO  (TEST-TestSimLargeCluster.testSearchRate-seed#[AE28E86C45BF8A88]) [    ] o.a.s.c.CloudTestUtils -- shards without leaders: [shard2]
   [junit4]   2> 3233381 INFO  (TEST-TestSimLargeCluster.testSearchRate-seed#[AE28E86C45BF8A88]) [    ] o.a.s.c.CloudTestUtils -- shards without leaders: [shard2]
   [junit4]   2> 3233384 INFO  (TEST-TestSimLargeCluster.testSearchRate-seed#[AE28E86C45BF8A88]) [    ] o.a.s.c.CloudTestUtils -- shards without leaders: [shard2]
   [junit4]   2> 3233386 INFO  (TEST-TestSimLargeCluster.testSearchRate-seed#[AE28E86C45BF8A88]) [    ] o.a.s.c.CloudTestUtils -- shards without leaders: [shard2]
   [junit4]   2> 3233387 DEBUG (simCloudManagerPool-8237-thread-19) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=18
   [junit4]   2> 3233396 DEBUG (simCloudManagerPool-8237-thread-19) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 18
   [junit4]   2> 3233406 DEBUG (simCloudManagerPool-8237-thread-20) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=19
   [junit4]   2> 3233407 DEBUG (simCloudManagerPool-8237-thread-20) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 19
   [junit4]   2> 3233440 DEBUG (simCloudManagerPool-8237-thread-21) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=20
   [junit4]   2> 3233456 DEBUG (simCloudManagerPool-8237-thread-21) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 20
   [junit4]   2> 3233489 DEBUG (simCloudManagerPool-8237-thread-37) [    ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testSearchRate / shard2 (currentVersion=21): {"core_node17":{
   [junit4]   2>     "core":"testSearchRate_shard2_replica_n17",
   [junit4]   2>     "shard":"shard2",
   [junit4]   2>     "collection":"testSearchRate",
   [junit4]   2>     "node_name":"127.0.0.1:10008_solr",
   [junit4]   2>     "type":"NRT",
   [junit4]   2>     "leader":"true",
   [junit4]   2>     "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>     "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>     "INDEX.sizeInBytes":10240,
   [junit4]   2>     "state":"active",
   [junit4]   2>     "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>     "SEARCHER.searcher.numDocs":0}}
   [junit4]   2> 3233500 DEBUG (simCloudManagerPool-8237-thread-22) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=21
   [junit4]   2> 3233500 DEBUG (simCloudManagerPool-8237-thread-22) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 21
   [junit4]   2> 3233517 DEBUG (simCloudManagerPool-8237-thread-23) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=22
   [junit4]   2> 3233522 DEBUG (simCloudManagerPool-8237-thread-23) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 22
   [junit4]   2> 3233549 DEBUG (simCloudManagerPool-8237-thread-24) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=23
   [junit4]   2> 3233550 DEBUG (simCloudManagerPool-8237-thread-24) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 23
   [junit4]   2> 3233570 DEBUG (simCloudManagerPool-8237-thread-9) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=24
   [junit4]   2> 3233570 DEBUG (simCloudManagerPool-8237-thread-9) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 24
   [junit4]   2> 3233571 INFO  (simCloudManagerPool-8237-thread-31) [    ] o.a.s.c.ActionThrottle Throttling leader attempts - waiting for 914ms
   [junit4]   2> 3233589 DEBUG (simCloudManagerPool-8237-thread-31) [    ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testSearchRate / shard2 (currentVersion=25): {"core_node11":{
   [junit4]   2>     "core":"testSearchRate_shard2_replica_n11",
   [junit4]   2>     "shard":"shard2",
   [junit4]   2>     "collection":"testSearchRate",
   [junit4]   2>     "node_name":"127.0.0.1:10031_solr",
   [junit4]   2>     "type":"NRT",
   [junit4]   2>     "leader":"true",
   [junit4]   2>     "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>     "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>     "INDEX.sizeInBytes":10240,
   [junit4]   2>     "state":"active",
   [junit4]   2>     "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>     "SEARCHER.searcher.numDocs":0}}
   [junit4]   2> 3233589 DEBUG (TEST-TestSimLargeCluster.testSearchRate-seed#[AE28E86C45BF8A88]) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=25
   [junit4]   2> 3233590 DEBUG (TEST-TestSimLargeCluster.testSearchRate-seed#[AE28E86C45BF8A88]) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 25
   [junit4]   2> 3233590 INFO  (TEST-TestSimLargeCluster.testSearchRate-seed#[AE28E86C45BF8A88]) [    ] o.a.s.c.a.s.TestSimLargeCluster Ready after 19239 ms
   [junit4]   2> 3233592 DEBUG (simCloudManagerPool-8237-thread-38) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=26
   [junit4]   2> 3233592 DEBUG (simCloudManagerPool-8237-thread-38) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 26
   [junit4]   2> 3233594 DEBUG (simCloudManagerPool-8237-thread-38) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 3233594 DEBUG (simCloudManagerPool-8237-thread-38) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 5
   [junit4]   2> 3233602 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 5
   [junit4]   2> 3233602 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
   [junit4]   2> 3233603 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 3233603 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 5, lastZnodeVersion 5
   [junit4]   2> 3233619 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10015_solr
   [junit4]   2> 3233621 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10069_solr
   [junit4]   2> 3233621 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10062_solr
   [junit4]   2> 3233621 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10008_solr
   [junit4]   2> 3233621 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10024_solr
   [junit4]   2> 3233621 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10050_solr
   [junit4]   2> 3233621 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10075_solr
   [junit4]   2> 3233621 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10078_solr
   [junit4]   2> 3233621 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10097_solr
   [junit4]   2> 3233621 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10031_solr
   [junit4]   2> 3233622 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10034_solr
   [junit4]   2> 3233622 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n10:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3233635 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10091_solr
   [junit4]   2> 3233635 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n9:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3233635 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10094_solr
   [junit4]   2> 3233635 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n8:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3233635 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10100_solr
   [junit4]   2> 3233635 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n7:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3233635 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10037_solr
   [junit4]   2> 3233635 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n6:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3233635 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10053_solr
   [junit4]   2> 3233635 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n5:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3233635 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10059_solr
   [junit4]   2> 3233635 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n4:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3233636 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10103_solr
   [junit4]   2> 3233636 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n3:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3233636 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10056_solr
   [junit4]   2> 3233636 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n2:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3233636 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10072_solr
   [junit4]   2> 3233636 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n1:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3233636 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ## Collection: testSearchRate
   [junit4]   2> 3233636 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##  - shard1
   [junit4]   2> 3233637 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n10  4.0
   [junit4]   2> 3233637 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n9  4.0
   [junit4]   2> 3233637 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n8  4.0
   [junit4]   2> 3233637 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n7  4.0
   [junit4]   2> 3233637 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n6  4.0
   [junit4]   2> 3233637 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n5  4.0
   [junit4]   2> 3233637 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n4  4.0
   [junit4]   2> 3233637 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n3  4.0
   [junit4]   2> 3233637 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n2  4.0
   [junit4]   2> 3233637 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n1  4.0
   [junit4]   2> 3233640 DEBUG (simCloudManagerPool-8237-thread-39) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 3233640 DEBUG (simCloudManagerPool-8237-thread-39) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 6
   [junit4]   2> 3233640 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 6
   [junit4]   2> 3233640 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
   [junit4]   2> 3233640 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 3233640 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion 6
   [junit4]   2> 3233730 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger -- testSearchRate.shard1: totalShardRate=40.0, shardRate=4.0
   [junit4]   2> 3233760 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10015_solr
   [junit4]   2> 3233760 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10069_solr
   [junit4]   2> 3233760 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10062_solr
   [junit4]   2> 3233760 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10008_solr
   [junit4]   2> 3233760 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10024_solr
   [junit4]   2> 3233761 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10050_solr
   [junit4]   2> 3233761 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10075_solr
   [junit4]   2> 3233761 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10078_solr
   [junit4]   2> 3233761 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10097_solr
   [junit4]   2> 3233761 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10031_solr
   [junit4]   2> 3233761 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10034_solr
   [junit4]   2> 3233761 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n10:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3233761 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10091_solr
   [junit4]   2> 3233761 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n9:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3233761 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10094_solr
   [junit4]   2> 3233761 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n8:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3233761 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10100_solr
   [junit4]   2> 3233761 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n7:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3233761 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10037_solr
   [junit4]   2> 3233761 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n6:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3233761 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10053_solr
   [junit4]   2> 3233761 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n5:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3233761 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10059_solr
   [junit4]   2> 3233761 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n4:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3233761 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10103_solr
   [junit4]   2> 3233761 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n3:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3233761 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10056_solr
   [junit4]   2> 3233761 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n2:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3233762 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10072_solr
   [junit4]   2> 3233762 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n1:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3233762 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ## Collection: testSearchRate
   [junit4]   2> 3233762 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##  - shard1
   [junit4]   2> 3233762 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n10  4.0
   [junit4]   2> 3233762 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n9  4.0
   [junit4]   2> 3233762 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n8  4.0
   [junit4]   2> 3233762 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n7  4.0
   [junit4]   2> 3233762 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n6  4.0
   [junit4]   2> 3233762 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n5  4.0
   [junit4]   2> 3233762 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n4  4.0
   [junit4]   2> 3233762 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n3  4.0
   [junit4]   2> 3233762 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n2  4.0
   [junit4]   2> 3233762 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n1  4.0
   [junit4]   2> 3233762 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.SearchRateTrigger -- testSearchRate.shard1: totalShardRate=40.0, shardRate=4.0
   [junit4]   2> 3233792 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"606618075e535dT5i4voylsylbqolmmspu1xw8hb",
   [junit4]   2>   "source":"search_rate_trigger",
   [junit4]   2>   "eventTime":27133851153093469,
   [junit4]   2>   "eventType":"SEARCHRATE",
   [junit4]   2>   "properties":{
   [junit4]   2>     "hotReplicas":[
   [junit4]   2>       {"core_node10":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n10",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10034_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node9":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n9",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10091_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node8":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n8",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10094_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node7":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n7",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10100_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node6":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n6",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10037_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node5":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n5",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10053_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node4":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n4",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10059_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node3":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n3",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10103_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node2":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n2",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10056_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node1":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n1",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10072_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "leader":"true",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}}],
   [junit4]   2>     "coldNodes":{},
   [junit4]   2>     "coldShards":{},
   [junit4]   2>     "coldReplicas":[],
   [junit4]   2>     "violationType":["hotShards"],
   [junit4]   2>     "coldCollections":{},
   [junit4]   2>     "hotShards":{"testSearchRate":{"shard1":4.0}},
   [junit4]   2>     "hotCollections":{"testSearchRate":40.0},
   [junit4]   2>     "hotNodes":{
   [junit4]   2>       "127.0.0.1:10094_solr":4.0,
   [junit4]   2>       "127.0.0.1:10091_solr":4.0,
   [junit4]   2>       "127.0.0.1:10034_solr":4.0,
   [junit4]   2>       "127.0.0.1:10100_solr":4.0,
   [junit4]   2>       "127.0.0.1:10037_solr":4.0,
   [junit4]   2>       "127.0.0.1:10053_solr":4.0,
   [junit4]   2>       "127.0.0.1:10059_solr":4.0,
   [junit4]   2>       "127.0.0.1:10103_solr":4.0,
   [junit4]   2>       "127.0.0.1:10072_solr":4.0,
   [junit4]   2>       "127.0.0.1:10056_solr":4.0},
   [junit4]   2>     "requestedOps":[{
   [junit4]   2>         "action":"ADDREPLICA",
   [junit4]   2>         "hints":{"COLL_SHARD":[{
   [junit4]   2>               "first":"testSearchRate",
   [junit4]   2>               "second":"shard1"}]}}]}}
   [junit4]   2> 3233829 DEBUG (ScheduledTrigger-8238-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [search_rate_trigger]
   [junit4]   2> 3233837 DEBUG (simCloudManagerPool-8237-thread-40) [    ] o.a.s.c.a.s.SimClusterStateProvider -- simCreateCollection .system, currentVersion=27
   [junit4]   2> 3233838 DEBUG (simCloudManagerPool-8237-thread-40) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=27
   [junit4]   2> 3233838 DEBUG (simCloudManagerPool-8237-thread-40) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 27
   [junit4]   2> 3233857 DEBUG (simCloudManagerPool-8237-thread-40) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=28
   [junit4]   2> 3233858 DEBUG (simCloudManagerPool-8237-thread-40) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 28
   [junit4]   2> 3234181 DEBUG (simCloudManagerPool-8237-thread-41) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=29
   [junit4]   2> 3234182 DEBUG (simCloudManagerPool-8237-thread-41) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 29
   [junit4]   2> 3234204 DEBUG (simCloudManagerPool-8237-thread-44) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=30
   [junit4]   2> 3234205 DEBUG (simCloudManagerPool-8237-thread-44) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 30
   [junit4]   2> 3234218 DEBUG (simCloudManagerPool-8237-thread-42) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=31
   [junit4]   2> 3234219 DEBUG (simCloudManagerPool-8237-thread-42) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 31
   [junit4]   2> 3234219 DEBUG (simCloudManagerPool-8237-thread-40) [    ] o.a.s.c.a.s.SimClusterStateProvider -- finished createCollection .system, currentVersion=32
   [junit4]   2> 3234219 INFO  (simCloudManagerPool-8237-thread-40) [    ] o.a.s.c.CloudTestUtils -- shards without leaders: [shard1]
   [junit4]   2> 3234219 DEBUG (simCloudManagerPool-8237-thread-43) [    ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for .system / shard1 (currentVersion=32): {"core_node1":{
   [junit4]   2>     "core":".system_shard1_replica_n1",
   [junit4]   2>     "shard":"shard1",
   [junit4]   2>     "collection":".system",
   [junit4]   2>     "node_name":"127.0.0.1:10063_solr",
   [junit4]   2>     "type":"NRT",
   [junit4]   2>     "leader":"true",
   [junit4]   2>     "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>     "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>     "INDEX.sizeInBytes":10240,
   [junit4]   2>     "state":"active",
   [junit4]   2>     "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>     "SEARCHER.searcher.numDocs":0}}
   [junit4]   2> 3234221 DEBUG (simCloudManagerPool-8237-thread-40) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=32
   [junit4]   2> 3234222 DEBUG (simCloudManagerPool-8237-thread-40) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 32
   [junit4]   2> 3234239 DEBUG (AutoscalingActionExecutor-8239-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"606618075e535dT5i4voylsylbqolmmspu1xw8hb",
   [junit4]   2>   "source":"search_rate_trigger",
   [junit4]   2>   "eventTime":27133851153093469,
   [junit4]   2>   "eventType":"SEARCHRATE",
   [junit4]   2>   "properties":{
   [junit4]   2>     "hotReplicas":[
   [junit4]   2>       {"core_node10":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n10",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10034_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node9":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n9",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10091_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node8":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n8",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10094_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node7":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n7",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10100_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node6":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n6",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10037_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node5":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n5",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10053_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node4":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n4",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10059_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node3":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n3",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10103_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node2":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n2",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10056_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node1":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n1",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10072_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "leader":"true",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}}],
   [junit4]   2>     "coldNodes":{},
   [junit4]   2>     "coldShards":{},
   [junit4]   2>     "coldReplicas":[],
   [junit4]   2>     "violationType":["hotShards"],
   [junit4]   2>     "coldCollections":{},
   [junit4]   2>     "hotShards":{"testSearchRate":{"shard1":4.0}},
   [junit4]   2>     "hotCollections":{"testSearchRate":40.0},
   [junit4]   2>     "hotNodes":{
   [junit4]   2>       "127.0.0.1:10094_solr":4.0,
   [junit4]   2>       "127.0.0.1:10091_solr":4.0,
   [junit4]   2>       "127.0.0.1:10034_solr":4.0,
   [junit4]   2>       "127.0.0.1:10100_solr":4.0,
   [junit4]   2>       "127.0.0.1:10037_solr":4.0,
   [junit4]   2>       "127.0.0.1:10053_solr":4.0,
   [junit4]   2>       "127.0.0.1:10059_solr":4.0,
   [junit4]   2>       "127.0.0.1:10103_solr":4.0,
   [junit4]   2>       "127.0.0.1:10072_solr":4.0,
   [junit4]   2>       "127.0.0.1:10056_solr":4.0},
   [junit4]   2>     "_enqueue_time_":27133860748358869,
   [junit4]   2>     "requestedOps":[{
   [junit4]   2>         "action":"ADDREPLICA",
   [junit4]   2>         "hints":{"COLL_SHARD":[{
   [junit4]   2>               "first":"testSearchRate",
   [junit4]   2>               "second":"shard1"}]}}]}}
   [junit4]   2> 3234240 DEBUG (AutoscalingActionExecutor-8239-thread-1) [    ] o.a.s.c.a.ComputePlanAction -- processing event: {
   [junit4]   2>   "id":"606618075e535dT5i4voylsylbqolmmspu1xw8hb",
   [junit4]   2>   "source":"search_rate_trigger",
   [junit4]   2>   "eventTime":27133851153093469,
   [junit4]   2>   "eventType":"SEARCHRATE",
   [junit4]   2>   "properties":{
   [junit4]   2>     "hotReplicas":[
   [junit4]   2>       {"core_node10":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n10",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10034_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node9":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n9",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10091_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node8":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n8",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10094_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node7":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n7",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10100_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node6":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n6",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10037_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node5":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n5",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10053_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node4":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n4",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10059_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node3":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n3",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10103_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node2":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n2",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10056_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node1":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n1",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10072_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "leader":"true",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}}],
   [junit4]   2>     "coldNodes":{},
   [junit4]   2>     "coldShards":{},
   [junit4]   2>     "coldReplicas":[],
   [junit4]   2>     "violationType":["hotShards"],
   [junit4]   2>     "coldCollections":{},
   [junit4]   2>     "hotShards":{"testSearchRate":{"shard1":4.0}},
   [junit4]   2>     "hotCollections":{"testSearchRate":40.0},
   [junit4]   2>     "hotNodes":{
   [junit4]   2>       "127.0.0.1:10094_solr":4.0,
   [junit4]   2>       "127.0.0.1:10091_solr":4.0,
   [junit4]   2>       "127.0.0.1:10034_solr":4.0,
   [junit4]   2>       "127.0.0.1:10100_solr":4.0,
   [junit4]   2>       "127.0.0.1:10037_solr":4.0,
   [junit4]   2>       "127.0.0.1:10053_solr":4.0,
   [junit4]   2>       "127.0.0.1:10059_solr":4.0,
   [junit4]   2>       "127.0.0.1:10103_solr":4.0,
   [junit4]   2>       "127.0.0.1:10072_solr":4.0,
   [junit4]   2>       "127.0.0.1:10056_solr":4.0},
   [junit4]   2>     "_enqueue_time_":27133860748358869,
   [junit4]   2>     "requestedOps":[{
   [junit4]   2>         "action":"ADDREPLICA",
   [junit4]   2>         "hints":{"COLL_SHARD":[{
   [junit4]   2>               "first":"testSearchRate",
   [junit4]   2>               "second":"shard1"}]}}]}} with context properties: {BEFORE_ACTION=[compute]}
   [junit4]   2> 3234292 DEBUG (AutoscalingActionExecutor-8239-thread-1) [    ] o.a.s.c.a.ComputePlanAction Computed Plan: action=ADDREPLICA&collection=testSearchRate&shard=shard1&node=127.0.0.1:10085_solr&type=NRT
   [junit4]   2> 3234292 DEBUG (AutoscalingActionExecutor-8239-thread-1) [    ] o.a.s.c.a.ExecutePlanAction -- processing event: {
   [junit4]   2>   "id":"606618075e535dT5i4voylsylbqolmmspu1xw8hb",
   [junit4]   2>   "source":"search_rate_trigger",
   [junit4]   2>   "eventTime":27133851153093469,
   [junit4]   2>   "eventType":"SEARCHRATE",
   [junit4]   2>   "properties":{
   [junit4]   2>     "hotReplicas":[
   [junit4]   2>       {"core_node10":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n10",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10034_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node9":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n9",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10091_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node8":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n8",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10094_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node7":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n7",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10100_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node6":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n6",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10037_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node5":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n5",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10053_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node4":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n4",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10059_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node3":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n3",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10103_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node2":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n2",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10056_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node1":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n1",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10072_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "leader":"true",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}}],
   [junit4]   2>     "__start__":1,
   [junit4]   2>     "coldNodes":{},
   [junit4]   2>     "coldShards":{},
   [junit4]   2>     "coldReplicas":[],
   [junit4]   2>     "violationType":["hotShards"],
   [junit4]   2>     "coldCollections":{},
   [junit4]   2>     "hotShards":{"testSearchRate":{"shard1":4.0}},
   [junit4]   2>     "hotCollections":{"testSearchRate":40.0},
   [junit4]   2>     "hotNodes":{
   [junit4]   2>       "127.0.0.1:10094_solr":4.0,
   [junit4]   2>       "127.0.0.1:10091_solr":4.0,
   [junit4]   2>       "127.0.0.1:10034_solr":4.0,
   [junit4]   2>       "127.0.0.1:10100_solr":4.0,
   [junit4]   2>       "127.0.0.1:10037_solr":4.0,
   [junit4]   2>       "127.0.0.1:10053_solr":4.0,
   [junit4]   2>       "127.0.0.1:10059_solr":4.0,
   [junit4]   2>       "127.0.0.1:10103_solr":4.0,
   [junit4]   2>       "127.0.0.1:10072_solr":4.0,
   [junit4]   2>       "127.0.0.1:10056_solr":4.0},
   [junit4]   2>     "_enqueue_time_":27133860748358869,
   [junit4]   2>     "requestedOps":[{
   [junit4]   2>         "action":"ADDREPLICA",
   [junit4]   2>         "hints":{"COLL_SHARD":[{
   [junit4]   2>               "first":"testSearchRate",
   [junit4]   2>               "second":"shard1"}]}}]}} with context properties: {operations=[{
   [junit4]   2>   "class":"org.apache.solr.client.solrj.request.CollectionAdminRequest$AddReplica",
   [junit4]   2>   "method":"GET",
   [junit4]   2>   "params.action":"ADDREPLICA",
   [junit4]   2>   "params.collection":"testSearchRate",
   [junit4]   2>   "params.shard":"shard1",
   [junit4]   2>   "params.node":"127.0.0.1:10085_solr",
   [junit4]   2>   "params.type":"NRT"}], AFTER_ACTION=[compute], BEFORE_ACTION=[compute, execute]}
   [junit4]   2> 3234312 DEBUG (AutoscalingActionExecutor-8239-thread-1) [    ] o.a.s.c.a.ExecutePlanAction Executing operation: action=ADDREPLICA&collection=testSearchRate&shard=sh

[...truncated too long message...]

ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/null2077359432
     [copy] Copying 240 files to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/null2077359432
   [delete] Deleting directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/null2077359432

check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] 	confs: [default]
[ivy:cachepath] 	found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] 	found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] 	found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] 	found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] 	found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] 	found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] 	found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] 	found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 21ms :: artifacts dl 1ms
	---------------------------------------------------------------------
	|                  |            modules            ||   artifacts   |
	|       conf       | number| search|dwnlded|evicted|| number|dwnlded|
	---------------------------------------------------------------------
	|      default     |   8   |   0   |   0   |   0   ||   8   |   0   |
	---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...

-jenkins-base:

BUILD SUCCESSFUL
Total time: 198 minutes 37 seconds
Archiving artifacts
java.lang.InterruptedException: no matches found within 10000
	at hudson.FilePath$34.hasMatch(FilePath.java:2678)
	at hudson.FilePath$34.invoke(FilePath.java:2557)
	at hudson.FilePath$34.invoke(FilePath.java:2547)
	at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2918)
Also:   hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene2
		at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
		at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
		at hudson.remoting.Channel.call(Channel.java:955)
		at hudson.FilePath.act(FilePath.java:1036)
		at hudson.FilePath.act(FilePath.java:1025)
		at hudson.FilePath.validateAntFileMask(FilePath.java:2547)
		at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
		at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
		at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
		at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
		at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
		at hudson.model.Build$BuildExecution.post2(Build.java:186)
		at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
		at hudson.model.Run.execute(Run.java:1819)
		at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
		at hudson.model.ResourceController.execute(ResourceController.java:97)
		at hudson.model.Executor.run(Executor.java:429)
Caused: hudson.FilePath$TunneledInterruptedException
	at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2920)
	at hudson.remoting.UserRequest.perform(UserRequest.java:212)
	at hudson.remoting.UserRequest.perform(UserRequest.java:54)
	at hudson.remoting.Request$2.run(Request.java:369)
	at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:748)
Caused: java.lang.InterruptedException: java.lang.InterruptedException: no matches found within 10000
	at hudson.FilePath.act(FilePath.java:1038)
	at hudson.FilePath.act(FilePath.java:1025)
	at hudson.FilePath.validateAntFileMask(FilePath.java:2547)
	at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
	at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
	at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
	at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
	at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
	at hudson.model.Build$BuildExecution.post2(Build.java:186)
	at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
	at hudson.model.Run.execute(Run.java:1819)
	at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
	at hudson.model.ResourceController.execute(ResourceController.java:97)
	at hudson.model.Executor.run(Executor.java:429)
No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)

[JENKINS] Lucene-Solr-Tests-master - Build # 3051 - Still Unstable

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-master/3051/

1 tests failed.
FAILED:  org.apache.solr.cloud.autoscaling.sim.TestSimLargeCluster.testSearchRate

Error Message:


Stack Trace:
java.util.ConcurrentModificationException
	at __randomizedtesting.SeedInfo.seed([BC783C8A49BC1668:E1302203867AB027]:0)
	at java.util.ArrayList$Itr.checkForComodification(ArrayList.java:909)
	at java.util.ArrayList$Itr.next(ArrayList.java:859)
	at java.util.AbstractCollection.toString(AbstractCollection.java:461)
	at java.lang.String.valueOf(String.java:2994)
	at java.lang.StringBuilder.append(StringBuilder.java:131)
	at java.util.concurrent.ConcurrentHashMap$MapEntry.toString(ConcurrentHashMap.java:3480)
	at java.lang.String.valueOf(String.java:2994)
	at java.lang.StringBuilder.append(StringBuilder.java:131)
	at java.util.concurrent.ConcurrentHashMap$CollectionView.toString(ConcurrentHashMap.java:4480)
	at org.apache.solr.cloud.autoscaling.sim.TestSimLargeCluster.testSearchRate(TestSimLargeCluster.java:702)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:935)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:971)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:985)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:944)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:830)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 14854 lines...]
   [junit4] Suite: org.apache.solr.cloud.autoscaling.sim.TestSimLargeCluster
   [junit4]   2> 3409671 INFO  (SUITE-TestSimLargeCluster-seed#[BC783C8A49BC1668]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.autoscaling.sim.TestSimLargeCluster_BC783C8A49BC1668-001/init-core-data-001
   [junit4]   2> 3409705 INFO  (SUITE-TestSimLargeCluster-seed#[BC783C8A49BC1668]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 3409707 INFO  (SUITE-TestSimLargeCluster-seed#[BC783C8A49BC1668]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 3409925 DEBUG (SUITE-TestSimLargeCluster-seed#[BC783C8A49BC1668]-worker) [    ] o.a.s.c.a.s.SimClusterStateProvider --- new Overseer leader: 127.0.0.1:10000_solr
   [junit4]   2> 3409926 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
   [junit4]   2> 3409926 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 0
   [junit4]   2> 3409943 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion -1
   [junit4]   2> 3409944 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 0
   [junit4]   2> 3409957 DEBUG (SUITE-TestSimLargeCluster-seed#[BC783C8A49BC1668]-worker) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=0
   [junit4]   2> 3410048 DEBUG (SUITE-TestSimLargeCluster-seed#[BC783C8A49BC1668]-worker) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 0
   [junit4]   2> 3410048 INFO  (SUITE-TestSimLargeCluster-seed#[BC783C8A49BC1668]-worker) [    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 3410060 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10000_solr]
   [junit4]   2> 3410080 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 3410080 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion 0
   [junit4]   2> 3410081 DEBUG (ScheduledTrigger-9799-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 46 and last live nodes: 2
   [junit4] IGNOR/A 0.00s J2 | TestSimLargeCluster.testBasic
   [junit4]    > Assumption #1: 'awaitsfix' test group is disabled (@AwaitsFix(bugUrl=https://issues.apache.org/jira/browse/SOLR-12028))
   [junit4]   2> 3410117 DEBUG (ScheduledTrigger-9799-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100 and last live nodes: 46
   [junit4]   2> 3410134 INFO  (TEST-TestSimLargeCluster.testSearchRate-seed#[BC783C8A49BC1668]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testSearchRate
   [junit4]   2> 3410134 DEBUG (TEST-TestSimLargeCluster.testSearchRate-seed#[BC783C8A49BC1668]) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
   [junit4]   2> 3410134 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
   [junit4]   2> 3410135 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
   [junit4]   2> 3410135 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 3410135 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
   [junit4]   2> 3410154 INFO  (TEST-TestSimLargeCluster.testSearchRate-seed#[BC783C8A49BC1668]) [    ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
   [junit4]   2> 3410154 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 3410155 DEBUG (TEST-TestSimLargeCluster.testSearchRate-seed#[BC783C8A49BC1668]) [    ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
   [junit4]   2> 3410155 DEBUG (TEST-TestSimLargeCluster.testSearchRate-seed#[BC783C8A49BC1668]) [    ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now
   [junit4]   2> 3410155 DEBUG (TEST-TestSimLargeCluster.testSearchRate-seed#[BC783C8A49BC1668]) [    ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
   [junit4]   2> 3410155 DEBUG (TEST-TestSimLargeCluster.testSearchRate-seed#[BC783C8A49BC1668]) [    ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor
   [junit4]   2> 3410155 DEBUG (TEST-TestSimLargeCluster.testSearchRate-seed#[BC783C8A49BC1668]) [    ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
   [junit4]   2> 3410155 DEBUG (TEST-TestSimLargeCluster.testSearchRate-seed#[BC783C8A49BC1668]) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 3410182 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
   [junit4]   2> 3410182 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2
   [junit4]   2> 3410182 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion -1
   [junit4]   2> 3410182 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
   [junit4]   2> 3410203 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10082_solr, 127.0.0.1:10004_solr, 127.0.0.1:10018_solr, 127.0.0.1:10049_solr, 127.0.0.1:10020_solr, 127.0.0.1:10065_solr, 127.0.0.1:10021_solr, 127.0.0.1:10005_solr, 127.0.0.1:10066_solr, 127.0.0.1:10035_solr, 127.0.0.1:10051_solr, 127.0.0.1:10019_solr, 127.0.0.1:10096_solr, 127.0.0.1:10022_solr, 127.0.0.1:10033_solr, 127.0.0.1:10017_solr, 127.0.0.1:10047_solr, 127.0.0.1:10095_solr, 127.0.0.1:10081_solr, 127.0.0.1:10084_solr, 127.0.0.1:10036_solr, 127.0.0.1:10052_solr, 127.0.0.1:10006_solr, 127.0.0.1:10079_solr, 127.0.0.1:10098_solr, 127.0.0.1:10003_solr, 127.0.0.1:10001_solr, 127.0.0.1:10054_solr, 127.0.0.1:10099_solr, 127.0.0.1:10071_solr, 127.0.0.1:10023_solr, 127.0.0.1:10076_solr, 127.0.0.1:10007_solr, 127.0.0.1:10055_solr, 127.0.0.1:10010_solr, 127.0.0.1:10016_solr, 127.0.0.1:10070_solr, 127.0.0.1:10077_solr, 127.0.0.1:10038_solr, 127.0.0.1:10032_solr, 127.0.0.1:10093_solr, 127.0.0.1:10061_solr, 127.0.0.1:10067_solr, 127.0.0.1:10086_solr, 127.0.0.1:10089_solr, 127.0.0.1:10045_solr, 127.0.0.1:10042_solr, 127.0.0.1:10029_solr, 127.0.0.1:10083_solr, 127.0.0.1:10080_solr, 127.0.0.1:10048_solr, 127.0.0.1:10064_solr, 127.0.0.1:10043_solr, 127.0.0.1:10057_solr, 127.0.0.1:10074_solr, 127.0.0.1:10026_solr, 127.0.0.1:10012_solr, 127.0.0.1:10060_solr, 127.0.0.1:10073_solr, 127.0.0.1:10058_solr, 127.0.0.1:10013_solr, 127.0.0.1:10090_solr, 127.0.0.1:10088_solr, 127.0.0.1:10027_solr, 127.0.0.1:10039_solr, 127.0.0.1:10025_solr, 127.0.0.1:10044_solr, 127.0.0.1:10028_solr, 127.0.0.1:10092_solr, 127.0.0.1:10041_solr, 127.0.0.1:10087_solr, 127.0.0.1:10009_solr, 127.0.0.1:10000_solr, 127.0.0.1:10030_solr, 127.0.0.1:10014_solr, 127.0.0.1:10011_solr, 127.0.0.1:10040_solr, 127.0.0.1:10085_solr, 127.0.0.1:10046_solr, 127.0.0.1:10068_solr, 127.0.0.1:10063_solr, 127.0.0.1:10015_solr, 127.0.0.1:10069_solr, 127.0.0.1:10062_solr, 127.0.0.1:10008_solr, 127.0.0.1:10024_solr, 127.0.0.1:10050_solr, 127.0.0.1:10075_solr, 127.0.0.1:10078_solr, 127.0.0.1:10002_solr, 127.0.0.1:10097_solr, 127.0.0.1:10031_solr, 127.0.0.1:10034_solr, 127.0.0.1:10091_solr, 127.0.0.1:10094_solr, 127.0.0.1:10037_solr, 127.0.0.1:10053_solr, 127.0.0.1:10059_solr, 127.0.0.1:10056_solr, 127.0.0.1:10072_solr]
   [junit4]   2> 3410210 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 3410211 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
   [junit4]   2> 3410211 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100 and last live nodes: 100
   [junit4]   2> 3410251 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100 and last live nodes: 100
   [junit4]   2> 3410275 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100 and last live nodes: 100
   [junit4]   2> 3410299 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100 and last live nodes: 100
   [junit4]   2> 3410328 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100 and last live nodes: 100
   [junit4]   2> 3410348 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100 and last live nodes: 100
   [junit4]   2> 3410368 DEBUG (ScheduledTrigger-9804-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100 and last live nodes: 100
   [junit4]   2> 3410382 DEBUG (simCloudManagerPool-9803-thread-1) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=1
   [junit4]   2> 3410382 DEBUG (simCloudManagerPool-9803-thread-1) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 1
   [junit4]   2> 3410398 DEBUG (simCloudManagerPool-9803-thread-1) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 3410398 DEBUG (simCloudManagerPool-9803-thread-1) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 3
   [junit4]   2> 3410430 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
   [junit4]   2> 3410430 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 3410430 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
   [junit4]   2> 3410445 DEBUG (ScheduledTrigger-9804-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100 and last live nodes: 100
   [junit4]   2> 3410447 DEBUG (simCloudManagerPool-9803-thread-2) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 3410447 DEBUG (simCloudManagerPool-9803-thread-2) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 4
   [junit4]   2> 3410451 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 4
   [junit4]   2> 3410451 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
   [junit4]   2> 3410451 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 3410451 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion 4
   [junit4]   2> 3410539 DEBUG (simCloudManagerPool-9803-thread-3) [    ] o.a.s.c.a.s.SimClusterStateProvider -- simCreateCollection testSearchRate, currentVersion=2
   [junit4]   2> 3410539 DEBUG (simCloudManagerPool-9803-thread-3) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=2
   [junit4]   2> 3410539 DEBUG (simCloudManagerPool-9803-thread-3) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 2
   [junit4]   2> 3410627 DEBUG (simCloudManagerPool-9803-thread-3) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=3
   [junit4]   2> 3410630 DEBUG (simCloudManagerPool-9803-thread-3) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 3
   [junit4]   2> 3412607 DEBUG (simCloudManagerPool-9803-thread-4) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=4
   [junit4]   2> 3412608 DEBUG (simCloudManagerPool-9803-thread-4) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 4
   [junit4]   2> 3412779 DEBUG (simCloudManagerPool-9803-thread-3) [    ] o.a.s.c.a.s.SimClusterStateProvider -- finished createCollection testSearchRate, currentVersion=5
   [junit4]   2> 3412873 DEBUG (simCloudManagerPool-9803-thread-8) [    ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testSearchRate / shard1 (currentVersion=5): {"core_node1":{
   [junit4]   2>     "core":"testSearchRate_shard1_replica_n1",
   [junit4]   2>     "shard":"shard1",
   [junit4]   2>     "collection":"testSearchRate",
   [junit4]   2>     "node_name":"127.0.0.1:10072_solr",
   [junit4]   2>     "type":"NRT",
   [junit4]   2>     "leader":"true",
   [junit4]   2>     "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>     "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>     "INDEX.sizeInBytes":10240,
   [junit4]   2>     "state":"active",
   [junit4]   2>     "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>     "SEARCHER.searcher.numDocs":0}}
   [junit4]   2> 3412901 DEBUG (simCloudManagerPool-9803-thread-5) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=5
   [junit4]   2> 3412901 DEBUG (simCloudManagerPool-9803-thread-5) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 5
   [junit4]   2> 3412921 DEBUG (simCloudManagerPool-9803-thread-6) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=6
   [junit4]   2> 3412937 DEBUG (simCloudManagerPool-9803-thread-6) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 6
   [junit4]   2> 3412961 DEBUG (simCloudManagerPool-9803-thread-7) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=7
   [junit4]   2> 3412962 DEBUG (simCloudManagerPool-9803-thread-7) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 7
   [junit4]   2> 3412998 DEBUG (simCloudManagerPool-9803-thread-10) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=8
   [junit4]   2> 3412999 DEBUG (simCloudManagerPool-9803-thread-10) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 8
   [junit4]   2> 3413022 DEBUG (simCloudManagerPool-9803-thread-9) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=9
   [junit4]   2> 3413023 DEBUG (simCloudManagerPool-9803-thread-9) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 9
   [junit4]   2> 3413042 DEBUG (simCloudManagerPool-9803-thread-11) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=10
   [junit4]   2> 3413132 DEBUG (simCloudManagerPool-9803-thread-11) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 10
   [junit4]   2> 3413160 DEBUG (simCloudManagerPool-9803-thread-12) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=11
   [junit4]   2> 3413180 DEBUG (simCloudManagerPool-9803-thread-12) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 11
   [junit4]   2> 3413212 DEBUG (simCloudManagerPool-9803-thread-13) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=12
   [junit4]   2> 3413213 DEBUG (simCloudManagerPool-9803-thread-13) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 12
   [junit4]   2> 3413229 DEBUG (simCloudManagerPool-9803-thread-14) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=13
   [junit4]   2> 3413229 DEBUG (simCloudManagerPool-9803-thread-14) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 13
   [junit4]   2> 3413249 DEBUG (simCloudManagerPool-9803-thread-15) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=14
   [junit4]   2> 3413249 DEBUG (simCloudManagerPool-9803-thread-15) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 14
   [junit4]   2> 3413265 DEBUG (simCloudManagerPool-9803-thread-25) [    ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testSearchRate / shard2 (currentVersion=15): {"core_node11":{
   [junit4]   2>     "core":"testSearchRate_shard2_replica_n11",
   [junit4]   2>     "shard":"shard2",
   [junit4]   2>     "collection":"testSearchRate",
   [junit4]   2>     "node_name":"127.0.0.1:10002_solr",
   [junit4]   2>     "type":"NRT",
   [junit4]   2>     "leader":"true",
   [junit4]   2>     "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>     "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>     "INDEX.sizeInBytes":10240,
   [junit4]   2>     "state":"active",
   [junit4]   2>     "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>     "SEARCHER.searcher.numDocs":0}}
   [junit4]   2> 3413289 DEBUG (simCloudManagerPool-9803-thread-16) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=15
   [junit4]   2> 3413290 DEBUG (simCloudManagerPool-9803-thread-16) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 15
   [junit4]   2> 3413314 DEBUG (simCloudManagerPool-9803-thread-17) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=16
   [junit4]   2> 3413315 DEBUG (simCloudManagerPool-9803-thread-17) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 16
   [junit4]   2> 3413334 DEBUG (simCloudManagerPool-9803-thread-18) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=17
   [junit4]   2> 3413424 DEBUG (simCloudManagerPool-9803-thread-18) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 17
   [junit4]   2> 3413443 DEBUG (simCloudManagerPool-9803-thread-19) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=18
   [junit4]   2> 3413444 DEBUG (simCloudManagerPool-9803-thread-19) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 18
   [junit4]   2> 3413460 DEBUG (simCloudManagerPool-9803-thread-20) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=19
   [junit4]   2> 3413481 DEBUG (simCloudManagerPool-9803-thread-20) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 19
   [junit4]   2> 3413496 DEBUG (simCloudManagerPool-9803-thread-21) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=20
   [junit4]   2> 3413500 DEBUG (simCloudManagerPool-9803-thread-21) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 20
   [junit4]   2> 3413516 DEBUG (simCloudManagerPool-9803-thread-22) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=21
   [junit4]   2> 3413537 DEBUG (simCloudManagerPool-9803-thread-22) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 21
   [junit4]   2> 3413553 DEBUG (simCloudManagerPool-9803-thread-23) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=22
   [junit4]   2> 3413642 DEBUG (simCloudManagerPool-9803-thread-23) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 22
   [junit4]   2> 3413662 DEBUG (simCloudManagerPool-9803-thread-24) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=23
   [junit4]   2> 3413679 DEBUG (simCloudManagerPool-9803-thread-24) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 23
   [junit4]   2> 3413690 DEBUG (TEST-TestSimLargeCluster.testSearchRate-seed#[BC783C8A49BC1668]) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=24
   [junit4]   2> 3413748 DEBUG (TEST-TestSimLargeCluster.testSearchRate-seed#[BC783C8A49BC1668]) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 24
   [junit4]   2> 3413748 INFO  (TEST-TestSimLargeCluster.testSearchRate-seed#[BC783C8A49BC1668]) [    ] o.a.s.c.a.s.TestSimLargeCluster Ready after 48291 ms
   [junit4]   2> 3413755 DEBUG (MetricsHistoryHandler-9802-thread-1) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=25
   [junit4]   2> 3413881 DEBUG (MetricsHistoryHandler-9802-thread-1) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 25
   [junit4]   2> 3413893 DEBUG (simCloudManagerPool-9803-thread-26) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=26
   [junit4]   2> 3413893 DEBUG (simCloudManagerPool-9803-thread-26) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 26
   [junit4]   2> 3413918 DEBUG (simCloudManagerPool-9803-thread-26) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 3413919 DEBUG (simCloudManagerPool-9803-thread-26) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 5
   [junit4]   2> 3413919 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 5
   [junit4]   2> 3413919 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
   [junit4]   2> 3413919 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 3413919 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 5, lastZnodeVersion 5
   [junit4]   2> 3413930 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10063_solr
   [junit4]   2> 3413932 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10015_solr
   [junit4]   2> 3413932 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10069_solr
   [junit4]   2> 3413932 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10062_solr
   [junit4]   2> 3413932 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10008_solr
   [junit4]   2> 3413932 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10024_solr
   [junit4]   2> 3413932 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10050_solr
   [junit4]   2> 3413932 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10075_solr
   [junit4]   2> 3413932 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10078_solr
   [junit4]   2> 3413932 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10002_solr
   [junit4]   2> 3413932 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10097_solr
   [junit4]   2> 3413932 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n10:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3413932 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10031_solr
   [junit4]   2> 3413932 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n9:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3413933 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10034_solr
   [junit4]   2> 3413933 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n8:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3413933 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10091_solr
   [junit4]   2> 3413933 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n7:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3413933 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10094_solr
   [junit4]   2> 3413933 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n6:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3413933 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10037_solr
   [junit4]   2> 3413933 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n5:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3413933 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10053_solr
   [junit4]   2> 3413933 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n4:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3413933 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10059_solr
   [junit4]   2> 3413933 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n3:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3413933 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10056_solr
   [junit4]   2> 3413933 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n2:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3413933 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10072_solr
   [junit4]   2> 3413933 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n1:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3413935 DEBUG (simCloudManagerPool-9803-thread-27) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 3413935 DEBUG (simCloudManagerPool-9803-thread-27) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 6
   [junit4]   2> 3413936 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 6
   [junit4]   2> 3413936 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
   [junit4]   2> 3413936 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 3413936 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion 6
   [junit4]   2> 3413982 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ## Collection: testSearchRate
   [junit4]   2> 3413983 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ##  - shard1
   [junit4]   2> 3413983 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n10  4.0
   [junit4]   2> 3413983 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n9  4.0
   [junit4]   2> 3413983 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n8  4.0
   [junit4]   2> 3413983 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n7  4.0
   [junit4]   2> 3413983 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n6  4.0
   [junit4]   2> 3413983 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n5  4.0
   [junit4]   2> 3413983 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n4  4.0
   [junit4]   2> 3413983 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n3  4.0
   [junit4]   2> 3413983 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n2  4.0
   [junit4]   2> 3413983 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n1  4.0
   [junit4]   2> 3413983 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger -- testSearchRate.shard1: totalShardRate=40.0, shardRate=4.0
   [junit4]   2> 3414004 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10063_solr
   [junit4]   2> 3414005 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10015_solr
   [junit4]   2> 3414005 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10069_solr
   [junit4]   2> 3414005 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10062_solr
   [junit4]   2> 3414005 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10008_solr
   [junit4]   2> 3414005 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10024_solr
   [junit4]   2> 3414005 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10050_solr
   [junit4]   2> 3414005 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10075_solr
   [junit4]   2> 3414005 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10078_solr
   [junit4]   2> 3414005 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10002_solr
   [junit4]   2> 3414005 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10097_solr
   [junit4]   2> 3414005 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n10:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3414005 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10031_solr
   [junit4]   2> 3414005 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n9:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3414005 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10034_solr
   [junit4]   2> 3414005 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n8:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3414005 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10091_solr
   [junit4]   2> 3414005 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n7:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3414005 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10094_solr
   [junit4]   2> 3414005 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n6:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3414006 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10037_solr
   [junit4]   2> 3414006 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n5:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3414006 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10053_solr
   [junit4]   2> 3414006 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n4:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3414006 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10059_solr
   [junit4]   2> 3414006 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n3:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3414006 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10056_solr
   [junit4]   2> 3414006 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n2:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3414006 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10072_solr
   [junit4]   2> 3414006 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n1:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3414006 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ## Collection: testSearchRate
   [junit4]   2> 3414006 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ##  - shard1
   [junit4]   2> 3414006 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n10  4.0
   [junit4]   2> 3414006 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n9  4.0
   [junit4]   2> 3414006 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n8  4.0
   [junit4]   2> 3414006 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n7  4.0
   [junit4]   2> 3414006 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n6  4.0
   [junit4]   2> 3414006 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n5  4.0
   [junit4]   2> 3414006 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n4  4.0
   [junit4]   2> 3414006 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n3  4.0
   [junit4]   2> 3414006 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n2  4.0
   [junit4]   2> 3414006 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n1  4.0
   [junit4]   2> 3414006 DEBUG (ScheduledTrigger-9804-thread-2) [    ] o.a.s.c.a.SearchRateTrigger -- testSearchRate.shard1: totalShardRate=40.0, shardRate=4.0
   [junit4]   2> 3414052 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10063_solr
   [junit4]   2> 3414052 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10015_solr
   [junit4]   2> 3414052 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10069_solr
   [junit4]   2> 3414052 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10062_solr
   [junit4]   2> 3414052 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10008_solr
   [junit4]   2> 3414052 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10024_solr
   [junit4]   2> 3414052 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10050_solr
   [junit4]   2> 3414052 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10075_solr
   [junit4]   2> 3414052 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10078_solr
   [junit4]   2> 3414052 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10002_solr
   [junit4]   2> 3414052 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10097_solr
   [junit4]   2> 3414052 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n10:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3414052 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10031_solr
   [junit4]   2> 3414052 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n9:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3414052 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10034_solr
   [junit4]   2> 3414052 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n8:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3414052 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10091_solr
   [junit4]   2> 3414052 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n7:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3414052 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10094_solr
   [junit4]   2> 3414053 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n6:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3414053 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10037_solr
   [junit4]   2> 3414053 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n5:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3414053 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10053_solr
   [junit4]   2> 3414053 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n4:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3414053 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10059_solr
   [junit4]   2> 3414053 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n3:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3414053 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10056_solr
   [junit4]   2> 3414053 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n2:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3414053 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10072_solr
   [junit4]   2> 3414053 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n1:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3414053 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ## Collection: testSearchRate
   [junit4]   2> 3414053 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##  - shard1
   [junit4]   2> 3414053 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n10  4.0
   [junit4]   2> 3414053 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n9  4.0
   [junit4]   2> 3414053 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n8  4.0
   [junit4]   2> 3414053 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n7  4.0
   [junit4]   2> 3414053 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n6  4.0
   [junit4]   2> 3414053 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n5  4.0
   [junit4]   2> 3414053 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n4  4.0
   [junit4]   2> 3414053 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n3  4.0
   [junit4]   2> 3414053 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n2  4.0
   [junit4]   2> 3414053 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n1  4.0
   [junit4]   2> 3414053 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger -- testSearchRate.shard1: totalShardRate=40.0, shardRate=4.0
   [junit4]   2> 3414075 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10063_solr
   [junit4]   2> 3414076 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10015_solr
   [junit4]   2> 3414076 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10069_solr
   [junit4]   2> 3414076 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10062_solr
   [junit4]   2> 3414076 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10008_solr
   [junit4]   2> 3414076 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10024_solr
   [junit4]   2> 3414076 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10050_solr
   [junit4]   2> 3414076 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10075_solr
   [junit4]   2> 3414076 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10078_solr
   [junit4]   2> 3414076 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10002_solr
   [junit4]   2> 3414076 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10097_solr
   [junit4]   2> 3414076 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n10:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3414076 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10031_solr
   [junit4]   2> 3414076 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n9:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3414076 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10034_solr
   [junit4]   2> 3414076 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n8:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3414076 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10091_solr
   [junit4]   2> 3414076 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n7:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3414076 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10094_solr
   [junit4]   2> 3414076 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n6:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3414076 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10037_solr
   [junit4]   2> 3414076 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n5:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3414077 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10053_solr
   [junit4]   2> 3414077 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n4:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3414077 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10059_solr
   [junit4]   2> 3414077 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n3:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3414077 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10056_solr
   [junit4]   2> 3414077 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n2:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3414077 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10072_solr
   [junit4]   2> 3414077 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n1:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3414077 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ## Collection: testSearchRate
   [junit4]   2> 3414077 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##  - shard1
   [junit4]   2> 3414077 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n10  4.0
   [junit4]   2> 3414077 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n9  4.0
   [junit4]   2> 3414077 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n8  4.0
   [junit4]   2> 3414077 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n7  4.0
   [junit4]   2> 3414077 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n6  4.0
   [junit4]   2> 3414077 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n5  4.0
   [junit4]   2> 3414077 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n4  4.0
   [junit4]   2> 3414077 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n3  4.0
   [junit4]   2> 3414077 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n2  4.0
   [junit4]   2> 3414077 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n1  4.0
   [junit4]   2> 3414077 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger -- testSearchRate.shard1: totalShardRate=40.0, shardRate=4.0
   [junit4]   2> 3414128 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10063_solr
   [junit4]   2> 3414128 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10015_solr
   [junit4]   2> 3414128 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10069_solr
   [junit4]   2> 3414128 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10062_solr
   [junit4]   2> 3414128 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10008_solr
   [junit4]   2> 3414128 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10024_solr
   [junit4]   2> 3414128 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10050_solr
   [junit4]   2> 3414128 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10075_solr
   [junit4]   2> 3414128 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10078_solr
   [junit4]   2> 3414128 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10002_solr
   [junit4]   2> 3414129 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10097_solr
   [junit4]   2> 3414129 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n10:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3414129 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10031_solr
   [junit4]   2> 3414129 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n9:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3414129 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10034_solr
   [junit4]   2> 3414129 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n8:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3414129 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10091_solr
   [junit4]   2> 3414129 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n7:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3414129 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10094_solr
   [junit4]   2> 3414129 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n6:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3414129 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10037_solr
   [junit4]   2> 3414129 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n5:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3414129 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10053_solr
   [junit4]   2> 3414129 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n4:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3414129 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10059_solr
   [junit4]   2> 3414129 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n3:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3414129 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10056_solr
   [junit4]   2> 3414129 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n2:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3414129 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ### rates for node 127.0.0.1:10072_solr
   [junit4]   2> 3414129 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ###  metrics:solr.core.testSearchRate.shard1.replica_n1:QUERY./select.requestTimes:1minRate	4
   [junit4]   2> 3414129 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ## Collection: testSearchRate
   [junit4]   2> 3414129 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##  - shard1
   [junit4]   2> 3414129 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n10  4.0
   [junit4]   2> 3414129 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n9  4.0
   [junit4]   2> 3414129 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n8  4.0
   [junit4]   2> 3414129 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n7  4.0
   [junit4]   2> 3414129 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n6  4.0
   [junit4]   2> 3414129 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n5  4.0
   [junit4]   2> 3414129 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n4  4.0
   [junit4]   2> 3414129 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n3  4.0
   [junit4]   2> 3414129 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n2  4.0
   [junit4]   2> 3414129 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger ##     testSearchRate_shard1_replica_n1  4.0
   [junit4]   2> 3414130 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.SearchRateTrigger -- testSearchRate.shard1: totalShardRate=40.0, shardRate=4.0
   [junit4]   2> 3414131 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"5f9ddbf990e9c5T96st6iqpml0dlnmmudjm7o4tt",
   [junit4]   2>   "source":"search_rate_trigger",
   [junit4]   2>   "eventTime":26913690897934789,
   [junit4]   2>   "eventType":"SEARCHRATE",
   [junit4]   2>   "properties":{
   [junit4]   2>     "hotReplicas":[
   [junit4]   2>       {"core_node10":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n10",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10097_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node9":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n9",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10031_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node8":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n8",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10034_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node7":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n7",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10091_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node6":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n6",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10094_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node5":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n5",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10037_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node4":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n4",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10053_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node3":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n3",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10059_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node2":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n2",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10056_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node1":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n1",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10072_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "leader":"true",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}}],
   [junit4]   2>     "coldNodes":{},
   [junit4]   2>     "coldShards":{},
   [junit4]   2>     "coldReplicas":[],
   [junit4]   2>     "violationType":["hotShards"],
   [junit4]   2>     "coldCollections":{},
   [junit4]   2>     "hotShards":{"testSearchRate":{"shard1":4.0}},
   [junit4]   2>     "hotCollections":{"testSearchRate":40.0},
   [junit4]   2>     "hotNodes":{
   [junit4]   2>       "127.0.0.1:10031_solr":4.0,
   [junit4]   2>       "127.0.0.1:10094_solr":4.0,
   [junit4]   2>       "127.0.0.1:10091_solr":4.0,
   [junit4]   2>       "127.0.0.1:10034_solr":4.0,
   [junit4]   2>       "127.0.0.1:10037_solr":4.0,
   [junit4]   2>       "127.0.0.1:10053_solr":4.0,
   [junit4]   2>       "127.0.0.1:10059_solr":4.0,
   [junit4]   2>       "127.0.0.1:10097_solr":4.0,
   [junit4]   2>       "127.0.0.1:10072_solr":4.0,
   [junit4]   2>       "127.0.0.1:10056_solr":4.0},
   [junit4]   2>     "requestedOps":[{
   [junit4]   2>         "action":"ADDREPLICA",
   [junit4]   2>         "hints":{"COLL_SHARD":[{
   [junit4]   2>               "first":"testSearchRate",
   [junit4]   2>               "second":"shard1"}]}}]}}
   [junit4]   2> 3414177 DEBUG (ScheduledTrigger-9804-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [search_rate_trigger]
   [junit4]   2> 3414195 DEBUG (simCloudManagerPool-9803-thread-28) [    ] o.a.s.c.a.s.SimClusterStateProvider -- simCreateCollection .system, currentVersion=27
   [junit4]   2> 3414195 DEBUG (simCloudManagerPool-9803-thread-28) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=27
   [junit4]   2> 3414215 DEBUG (simCloudManagerPool-9803-thread-28) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 27
   [junit4]   2> 3414241 DEBUG (simCloudManagerPool-9803-thread-28) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=28
   [junit4]   2> 3414242 DEBUG (simCloudManagerPool-9803-thread-28) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 28
   [junit4]   2> 3414477 DEBUG (simCloudManagerPool-9803-thread-30) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=29
   [junit4]   2> 3414478 DEBUG (simCloudManagerPool-9803-thread-30) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 29
   [junit4]   2> 3414493 DEBUG (simCloudManagerPool-9803-thread-29) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=30
   [junit4]   2> 3414549 DEBUG (simCloudManagerPool-9803-thread-29) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 30
   [junit4]   2> 3414586 DEBUG (simCloudManagerPool-9803-thread-31) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=31
   [junit4]   2> 3414821 DEBUG (simCloudManagerPool-9803-thread-31) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 31
   [junit4]   2> 3414821 DEBUG (simCloudManagerPool-9803-thread-28) [    ] o.a.s.c.a.s.SimClusterStateProvider -- finished createCollection .system, currentVersion=32
   [junit4]   2> 3414821 DEBUG (simCloudManagerPool-9803-thread-32) [    ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for .system / shard1 (currentVersion=32): {"core_node2":{
   [junit4]   2>     "core":".system_shard1_replica_n2",
   [junit4]   2>     "shard":"shard1",
   [junit4]   2>     "collection":".system",
   [junit4]   2>     "node_name":"127.0.0.1:10046_solr",
   [junit4]   2>     "type":"NRT",
   [junit4]   2>     "leader":"true",
   [junit4]   2>     "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>     "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>     "INDEX.sizeInBytes":10240,
   [junit4]   2>     "state":"active",
   [junit4]   2>     "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>     "SEARCHER.searcher.numDocs":0}}
   [junit4]   2> 3414821 INFO  (simCloudManagerPool-9803-thread-28) [    ] o.a.s.c.CloudTestUtils -- shards without leaders: [shard1]
   [junit4]   2> 3414878 DEBUG (simCloudManagerPool-9803-thread-28) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=32
   [junit4]   2> 3414879 DEBUG (simCloudManagerPool-9803-thread-28) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 32
   [junit4]   2> 3414883 DEBUG (AutoscalingActionExecutor-9805-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"5f9ddbf990e9c5T96st6iqpml0dlnmmudjm7o4tt",
   [junit4]   2>   "source":"search_rate_trigger",
   [junit4]   2>   "eventTime":26913690897934789,
   [junit4]   2>   "eventType":"SEARCHRATE",
   [junit4]   2>   "properties":{
   [junit4]   2>     "hotReplicas":[
   [junit4]   2>       {"core_node10":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n10",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10097_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node9":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n9",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10031_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node8":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n8",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10034_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node7":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n7",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10091_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node6":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n6",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10094_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node5":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n5",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10037_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node4":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n4",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10053_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node3":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n3",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10059_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node2":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n2",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10056_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node1":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n1",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10072_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "leader":"true",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}}],
   [junit4]   2>     "coldNodes":{},
   [junit4]   2>     "coldShards":{},
   [junit4]   2>     "coldReplicas":[],
   [junit4]   2>     "violationType":["hotShards"],
   [junit4]   2>     "coldCollections":{},
   [junit4]   2>     "hotShards":{"testSearchRate":{"shard1":4.0}},
   [junit4]   2>     "hotCollections":{"testSearchRate":40.0},
   [junit4]   2>     "hotNodes":{
   [junit4]   2>       "127.0.0.1:10031_solr":4.0,
   [junit4]   2>       "127.0.0.1:10094_solr":4.0,
   [junit4]   2>       "127.0.0.1:10091_solr":4.0,
   [junit4]   2>       "127.0.0.1:10034_solr":4.0,
   [junit4]   2>       "127.0.0.1:10037_solr":4.0,
   [junit4]   2>       "127.0.0.1:10053_solr":4.0,
   [junit4]   2>       "127.0.0.1:10059_solr":4.0,
   [junit4]   2>       "127.0.0.1:10097_solr":4.0,
   [junit4]   2>       "127.0.0.1:10072_solr":4.0,
   [junit4]   2>       "127.0.0.1:10056_solr":4.0},
   [junit4]   2>     "_enqueue_time_":26913700587590339,
   [junit4]   2>     "requestedOps":[{
   [junit4]   2>         "action":"ADDREPLICA",
   [junit4]   2>         "hints":{"COLL_SHARD":[{
   [junit4]   2>               "first":"testSearchRate",
   [junit4]   2>               "second":"shard1"}]}}]}}
   [junit4]   2> 3414884 DEBUG (AutoscalingActionExecutor-9805-thread-1) [    ] o.a.s.c.a.ComputePlanAction -- processing event: {
   [junit4]   2>   "id":"5f9ddbf990e9c5T96st6iqpml0dlnmmudjm7o4tt",
   [junit4]   2>   "source":"search_rate_trigger",
   [junit4]   2>   "eventTime":26913690897934789,
   [junit4]   2>   "eventType":"SEARCHRATE",
   [junit4]   2>   "properties":{
   [junit4]   2>     "hotReplicas":[
   [junit4]   2>       {"core_node10":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n10",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10097_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node9":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n9",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10031_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node8":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n8",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10034_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node7":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n7",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10091_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node6":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n6",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10094_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node5":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n5",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10037_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node4":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n4",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10053_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,
   [junit4]   2>           "rate":4.0,
   [junit4]   2>           "state":"active",
   [junit4]   2>           "QUERY./select.requestTimes:1minRate":4,
   [junit4]   2>           "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>           "SEARCHER.searcher.numDocs":0}},
   [junit4]   2>       {"core_node3":{
   [junit4]   2>           "core":"testSearchRate_shard1_replica_n3",
   [junit4]   2>           "shard":"shard1",
   [junit4]   2>           "collection":"testSearchRate",
   [junit4]   2>           "node_name":"127.0.0.1:10059_solr",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>           "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>           "INDEX.sizeInBytes":10240,


[...truncated too long message...]

ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/null1637257934
     [copy] Copying 240 files to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/null1637257934
   [delete] Deleting directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/null1637257934

check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] 	confs: [default]
[ivy:cachepath] 	found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] 	found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] 	found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] 	found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] 	found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] 	found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] 	found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] 	found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 28ms :: artifacts dl 1ms
	---------------------------------------------------------------------
	|                  |            modules            ||   artifacts   |
	|       conf       | number| search|dwnlded|evicted|| number|dwnlded|
	---------------------------------------------------------------------
	|      default     |   8   |   0   |   0   |   0   ||   8   |   0   |
	---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...

-jenkins-base:

BUILD SUCCESSFUL
Total time: 205 minutes 18 seconds
Archiving artifacts
java.lang.InterruptedException: no matches found within 10000
	at hudson.FilePath$34.hasMatch(FilePath.java:2678)
	at hudson.FilePath$34.invoke(FilePath.java:2557)
	at hudson.FilePath$34.invoke(FilePath.java:2547)
	at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2918)
Also:   hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene2
		at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
		at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
		at hudson.remoting.Channel.call(Channel.java:955)
		at hudson.FilePath.act(FilePath.java:1036)
		at hudson.FilePath.act(FilePath.java:1025)
		at hudson.FilePath.validateAntFileMask(FilePath.java:2547)
		at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
		at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
		at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
		at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
		at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
		at hudson.model.Build$BuildExecution.post2(Build.java:186)
		at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
		at hudson.model.Run.execute(Run.java:1819)
		at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
		at hudson.model.ResourceController.execute(ResourceController.java:97)
		at hudson.model.Executor.run(Executor.java:429)
Caused: hudson.FilePath$TunneledInterruptedException
	at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2920)
	at hudson.remoting.UserRequest.perform(UserRequest.java:212)
	at hudson.remoting.UserRequest.perform(UserRequest.java:54)
	at hudson.remoting.Request$2.run(Request.java:369)
	at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:748)
Caused: java.lang.InterruptedException: java.lang.InterruptedException: no matches found within 10000
	at hudson.FilePath.act(FilePath.java:1038)
	at hudson.FilePath.act(FilePath.java:1025)
	at hudson.FilePath.validateAntFileMask(FilePath.java:2547)
	at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
	at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
	at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
	at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
	at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
	at hudson.model.Build$BuildExecution.post2(Build.java:186)
	at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
	at hudson.model.Run.execute(Run.java:1819)
	at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
	at hudson.model.ResourceController.execute(ResourceController.java:97)
	at hudson.model.Executor.run(Executor.java:429)
No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)