You are viewing a plain text version of this content. The canonical link for it is here.
Posted to builds@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2021/03/02 15:45:17 UTC
[JENKINS] Lucene » Lucene-Solr-Tests-8.8 - Build # 366 - Unstable!
Build: https://ci-builds.apache.org/job/Lucene/job/Lucene-Solr-Tests-8.8/366/
2 tests failed.
FAILED: org.apache.solr.cloud.autoscaling.sim.TestSimScenario.testAutoAddReplicas
Error Message:
Timed out waiting for trigger .auto_add_replicas to fire after simulated 60s (real 1200ms).
Stack Trace:
java.io.IOException: Timed out waiting for trigger .auto_add_replicas to fire after simulated 60s (real 1200ms).
at __randomizedtesting.SeedInfo.seed([7B42D9418824EC48:F2E22A3DFC2F2061]:0)
at org.apache.solr.cloud.autoscaling.sim.SimScenario$SimWaitListener.wait(SimScenario.java:705)
at org.apache.solr.cloud.autoscaling.sim.SimScenario$WaitEvent.execute(SimScenario.java:771)
at org.apache.solr.cloud.autoscaling.sim.SimScenario.run(SimScenario.java:1119)
at org.apache.solr.cloud.autoscaling.sim.TestSimScenario.testAutoAddReplicas(TestSimScenario.java:57)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.metrics.reporters.SolrJmxReporterTest.testClosedCore
Error Message:
Captured an uncaught exception in thread: Thread[id=35235, name=Thread-4780, state=RUNNABLE, group=TGRP-SolrJmxReporterTest]
Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=35235, name=Thread-4780, state=RUNNABLE, group=TGRP-SolrJmxReporterTest]
at __randomizedtesting.SeedInfo.seed([7B42D9418824EC48:C3F0F61A8BD8EB8]:0)
Caused by: java.lang.AssertionError
at __randomizedtesting.SeedInfo.seed([7B42D9418824EC48]:0)
at org.junit.Assert.fail(Assert.java:87)
at org.junit.Assert.assertTrue(Assert.java:42)
at org.junit.Assert.assertNotNull(Assert.java:713)
at org.junit.Assert.assertNotNull(Assert.java:723)
at org.apache.solr.metrics.reporters.SolrJmxReporterTest$1.run(SolrJmxReporterTest.java:213)
Build Log:
[...truncated 13648 lines...]
[junit4] Suite: org.apache.solr.cloud.autoscaling.sim.TestSimScenario
[junit4] 2> 291189 INFO (SUITE-TestSimScenario-seed#[7B42D9418824EC48]-worker) [ ] o.a.s.SolrTestCase Setting 'solr.default.confdir' system property to test-framework derived value of '/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-Tests-8.8/solr/server/solr/configsets/_default/conf'
[junit4] 2> 291190 INFO (SUITE-TestSimScenario-seed#[7B42D9418824EC48]-worker) [ ] o.a.s.SolrTestCaseJ4 Created dataDir: /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-Tests-8.8/solr/build/solr-core/test/J3/temp/solr.cloud.autoscaling.sim.TestSimScenario_7B42D9418824EC48-001/data-dir-33-001
[junit4] 2> 291190 WARN (SUITE-TestSimScenario-seed#[7B42D9418824EC48]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=34 numCloses=34
[junit4] 2> 291190 INFO (SUITE-TestSimScenario-seed#[7B42D9418824EC48]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 291192 INFO (SUITE-TestSimScenario-seed#[7B42D9418824EC48]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 291192 INFO (SUITE-TestSimScenario-seed#[7B42D9418824EC48]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 291195 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[7B42D9418824EC48]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testAutoAddReplicas
[junit4] 2> 291215 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario 1. CreateCluster numNodes=2
[junit4] 2> 291216 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario CreateCluster numNodes=2
[junit4] 2> 291235 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[7B42D9418824EC48]) [ ] o.a.s.c.SolrPaths solr home defaulted to 'solr/' (could not find system property or JNDI)
[junit4] 2> 291258 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
[junit4] 2> 291258 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
[junit4] 2> 291258 DEBUG (TEST-TestSimScenario.testAutoAddReplicas-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimClusterStateProvider --- new Overseer leader: 127.0.0.1:10000_solr
[junit4] 2> 291258 DEBUG (TEST-TestSimScenario.testAutoAddReplicas-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=0
[junit4] 2> 291259 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 0
[junit4] 2> 291259 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
[junit4] 2> 291259 DEBUG (TEST-TestSimScenario.testAutoAddReplicas-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 0
[junit4] 2> 291259 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[7B42D9418824EC48]) [ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 291259 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario 2. LoadAutoscaling json={'cluster-policy'+:+[{'replica'+:+'<3',+'shard'+:+'#EACH',+'collection'+:+'testCollection','node':'#ANY'}]}&defaultWaitFor=10
[junit4] 2> 291260 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario LoadAutoscaling json={'cluster-policy'+:+[{'replica'+:+'<3',+'shard'+:+'#EACH',+'collection'+:+'testCollection','node':'#ANY'}]}&defaultWaitFor=10
[junit4] 2> 291260 DEBUG (TEST-TestSimScenario.testAutoAddReplicas-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2
[junit4] 2> 291261 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10000_solr]
[junit4] 2> 291261 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- deactivating old nodeLost / nodeAdded markers
[junit4] 2> 291261 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 1
[junit4] 2> 291261 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
[junit4] 2> 291261 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario 3. RunSolrRequest path=/admin/collections&action=CREATE&autoAddReplicas=true&name=testCollection&numShards=2&replicationFactor=2&maxShardsPerNode=2
[junit4] 2> 291261 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario RunSolrRequest path=/admin/collections&action=CREATE&autoAddReplicas=true&name=testCollection&numShards=2&replicationFactor=2&maxShardsPerNode=2
[junit4] 2> 291261 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10000_solr]
[junit4] 2> 291262 DEBUG (ScheduledTrigger-2910-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 291262 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- deactivating old nodeLost / nodeAdded markers
[junit4] 2> 291262 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
[junit4] 2> 291264 DEBUG (simCloudManagerPool-2909-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider -- simCreateCollection testCollection, currentVersion=1
[junit4] 2> 291264 DEBUG (simCloudManagerPool-2909-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=1
[junit4] 2> 291264 DEBUG (simCloudManagerPool-2909-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 1
[junit4] 2> 291275 DEBUG (simCloudManagerPool-2909-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=2
[junit4] 2> 291276 DEBUG (simCloudManagerPool-2909-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 2
[junit4] 2> 291286 DEBUG (ScheduledTrigger-2910-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 291314 DEBUG (ScheduledTrigger-2910-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 291334 DEBUG (ScheduledTrigger-2910-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 291354 DEBUG (ScheduledTrigger-2910-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 291372 DEBUG (simCloudManagerPool-2909-thread-2) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=3
[junit4] 2> 291373 DEBUG (simCloudManagerPool-2909-thread-2) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 3
[junit4] 2> 291373 DEBUG (simCloudManagerPool-2909-thread-2) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election (testCollection / shard1)
[junit4] 2> 291374 DEBUG (ScheduledTrigger-2910-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 291374 DEBUG (simCloudManagerPool-2909-thread-2) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testCollection / shard1 (currentVersion=4): {"core_node1":{}}
[junit4] 2> 291386 DEBUG (simCloudManagerPool-2909-thread-3) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=4
[junit4] 2> 291386 DEBUG (simCloudManagerPool-2909-thread-3) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 4
[junit4] 2> 291386 DEBUG (simCloudManagerPool-2909-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider -- finished createCollection testCollection, currentVersion=5
[junit4] 2> 291386 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario 4. WaitCollection collection=testCollection&shards=2&replicas=2
[junit4] 2> 291386 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario WaitCollection collection=testCollection&shards=2&replicas=2
[junit4] 2> 291394 DEBUG (ScheduledTrigger-2910-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 291396 DEBUG (simCloudManagerPool-2909-thread-4) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=5
[junit4] 2> 291396 DEBUG (simCloudManagerPool-2909-thread-4) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 5
[junit4] 2> 291396 DEBUG (simCloudManagerPool-2909-thread-4) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election (testCollection / shard2)
[junit4] 2> 291396 DEBUG (simCloudManagerPool-2909-thread-4) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testCollection / shard2 (currentVersion=6): {"core_node3":{}}
[junit4] 2> 291407 DEBUG (simCloudManagerPool-2909-thread-5) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=6
[junit4] 2> 291407 DEBUG (simCloudManagerPool-2909-thread-5) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 6
[junit4] 2> 291407 DEBUG (TEST-TestSimScenario.testAutoAddReplicas-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=7
[junit4] 2> 291407 DEBUG (TEST-TestSimScenario.testAutoAddReplicas-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 7
[junit4] 2> 291407 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario 5. SetEventListener trigger=.auto_add_replicas&stage=SUCCEEDED
[junit4] 2> 291407 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario SetEventListener trigger=.auto_add_replicas&stage=SUCCEEDED
[junit4] 2> 291408 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario 6. KillNodes node=${_random_node_}
[junit4] 2> 291408 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario KillNodes node=127.0.0.1:10001_solr
[junit4] 2> 291408 DEBUG (TEST-TestSimScenario.testAutoAddReplicas-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimClusterStateProvider -- created marker: /autoscaling/nodeLost/127.0.0.1:10001_solr
[junit4] 2> 291408 DEBUG (TEST-TestSimScenario.testAutoAddReplicas-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=8
[junit4] 2> 291409 DEBUG (TEST-TestSimScenario.testAutoAddReplicas-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 8
[junit4] 2> 291410 DEBUG (simCloudManagerPool-2909-thread-6) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election (testCollection / shard2)
[junit4] 2> 291410 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario 7. WaitEvent trigger=.auto_add_replicas&wait=60
[junit4] 2> 291410 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario WaitEvent trigger=.auto_add_replicas&wait=60
[junit4] 2> 291410 INFO (simCloudManagerPool-2909-thread-6) [ ] o.a.s.c.ActionThrottle Throttling leader attempts - waiting for 4333ms
[junit4] 2> 291419 DEBUG (ScheduledTrigger-2910-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 2
[junit4] 2> 291419 DEBUG (ScheduledTrigger-2910-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10001_solr
[junit4] 2> 291439 DEBUG (ScheduledTrigger-2910-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
[junit4] 2> 291459 DEBUG (ScheduledTrigger-2910-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
[junit4] 2> 291479 DEBUG (ScheduledTrigger-2910-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
[junit4] 2> 291497 DEBUG (simCloudManagerPool-2909-thread-6) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testCollection / shard2 (currentVersion=9): {"core_node4":{}}
[junit4] 2> 291497 DEBUG (simCloudManagerPool-2909-thread-7) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=9
[junit4] 2> 291497 DEBUG (simCloudManagerPool-2909-thread-7) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 9
[junit4] 2> 291499 DEBUG (ScheduledTrigger-2910-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
[junit4] 2> 291519 DEBUG (ScheduledTrigger-2910-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
[junit4] 2> 291539 DEBUG (ScheduledTrigger-2910-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
[junit4] 2> 291560 DEBUG (ScheduledTrigger-2910-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
[junit4] 2> 291580 DEBUG (ScheduledTrigger-2910-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
[junit4] 2> 291600 DEBUG (ScheduledTrigger-2910-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
[junit4] 2> 291620 DEBUG (ScheduledTrigger-2910-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
[junit4] 2> 292505 DEBUG (ScheduledTrigger-2910-thread-2) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:10001_solr]
[junit4] 2> 292459 DEBUG (MetricsHistoryHandler-2913-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=10
[junit4] 2> 292506 DEBUG (MetricsHistoryHandler-2913-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 10
[junit4] 2> 292506 DEBUG (ScheduledTrigger-2910-thread-2) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
[junit4] 2> "id":"37ede5dafd8229Tf25b6mipzt86fio5b3ofk6amg",
[junit4] 2> "source":".auto_add_replicas",
[junit4] 2> "eventTime":15742695196426793,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[15742695196426793],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "nodeNames":["127.0.0.1:10001_solr"]}}
[junit4] 2> 292507 DEBUG (ScheduledTrigger-2910-thread-2) [ ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, .scheduled_maintenance]
[junit4] 2> 292507 DEBUG (ScheduledTrigger-2910-thread-2) [ ] o.a.s.c.a.SystemLogListener Collection .system missing, skip sending event {
[junit4] 2> "id":"37ede5dafd8229Tf25b6mipzt86fio5b3ofk6amg",
[junit4] 2> "source":".auto_add_replicas",
[junit4] 2> "eventTime":15742695196426793,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[15742695196426793],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":15742749611684543,
[junit4] 2> "nodeNames":["127.0.0.1:10001_solr"]}}
[junit4] 2> 292508 DEBUG (AutoscalingActionExecutor-2911-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
[junit4] 2> "id":"37ede5dafd8229Tf25b6mipzt86fio5b3ofk6amg",
[junit4] 2> "source":".auto_add_replicas",
[junit4] 2> "eventTime":15742695196426793,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[15742695196426793],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":15742749611684543,
[junit4] 2> "nodeNames":["127.0.0.1:10001_solr"]}}
[junit4] 2> 292509 DEBUG (AutoscalingActionExecutor-2911-thread-1) [ ] o.a.s.c.a.ComputePlanAction -- processing event: {
[junit4] 2> "id":"37ede5dafd8229Tf25b6mipzt86fio5b3ofk6amg",
[junit4] 2> "source":".auto_add_replicas",
[junit4] 2> "eventTime":15742695196426793,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[15742695196426793],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":15742749611684543,
[junit4] 2> "nodeNames":["127.0.0.1:10001_solr"]}} with context properties: {BEFORE_ACTION=[auto_add_replicas_plan]}
[junit4] 2> 292509 DEBUG (AutoscalingActionExecutor-2911-thread-1) [ ] o.a.s.c.a.ComputePlanAction -- estimated total max ops=12, effective maxOps=12
[junit4] 2> 292510 DEBUG (AutoscalingActionExecutor-2911-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testCollection&targetNode=127.0.0.1:10000_solr&inPlaceMove=true&replica=core_node2
[junit4] 2> 292510 DEBUG (AutoscalingActionExecutor-2911-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testCollection&targetNode=127.0.0.1:10000_solr&inPlaceMove=true&replica=core_node3
[junit4] 2> 292510 DEBUG (AutoscalingActionExecutor-2911-thread-1) [ ] o.a.s.c.a.ComputePlanAction -- no more operations suggested, stopping after 2 ops...
[junit4] 2> 292510 DEBUG (AutoscalingActionExecutor-2911-thread-1) [ ] o.a.s.c.a.ExecutePlanAction -- processing event: {
[junit4] 2> "id":"37ede5dafd8229Tf25b6mipzt86fio5b3ofk6amg",
[junit4] 2> "source":".auto_add_replicas",
[junit4] 2> "eventTime":15742695196426793,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[15742695196426793],
[junit4] 2> "__compute_diag__":{
[junit4] 2> "noSuggestionsStopAfter":2,
[junit4] 2> "estimatedMaxOps":12,
[junit4] 2> "requestedOperations":-1,
[junit4] 2> "opLimit":12,
[junit4] 2> "maxOperations":12},
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":15742749611684543,
[junit4] 2> "nodeNames":["127.0.0.1:10001_solr"]}} with context properties: {operations=[{
[junit4] 2> "class":"org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica",
[junit4] 2> "method":"GET",
[junit4] 2> "params.action":"MOVEREPLICA",
[junit4] 2> "params.collection":"testCollection",
[junit4] 2> "params.targetNode":"127.0.0.1:10000_solr",
[junit4] 2> "params.inPlaceMove":"true",
[junit4] 2> "params.replica":"core_node2"}, {
[junit4] 2> "class":"org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica",
[junit4] 2> "method":"GET",
[junit4] 2> "params.action":"MOVEREPLICA",
[junit4] 2> "params.collection":"testCollection",
[junit4] 2> "params.targetNode":"127.0.0.1:10000_solr",
[junit4] 2> "params.inPlaceMove":"true",
[junit4] 2> "params.replica":"core_node3"}], AFTER_ACTION=[auto_add_replicas_plan], BEFORE_ACTION=[auto_add_replicas_plan, execute_plan]}
[junit4] 2> 292510 DEBUG (AutoscalingActionExecutor-2911-thread-1) [ ] o.a.s.c.a.ExecutePlanAction Executing operation: action=MOVEREPLICA&collection=testCollection&targetNode=127.0.0.1:10000_solr&inPlaceMove=true&replica=core_node2
[junit4] 2> 292611 DEBUG (AutoscalingActionExecutor-2911-thread-1) [ ] o.a.s.c.a.SystemLogListener Collection .system missing, skip sending event {
[junit4] 2> "id":"37ede5dafd8229Tf25b6mipzt86fio5b3ofk6amg",
[junit4] 2> "source":".auto_add_replicas",
[junit4] 2> "eventTime":15742695196426793,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[15742695196426793],
[junit4] 2> "__compute_diag__":{
[junit4] 2> "noSuggestionsStopAfter":2,
[junit4] 2> "estimatedMaxOps":12,
[junit4] 2> "requestedOperations":-1,
[junit4] 2> "opLimit":12,
[junit4] 2> "maxOperations":12},
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":15742749611684543,
[junit4] 2> "nodeNames":["127.0.0.1:10001_solr"]}}
[junit4] 2> 292612 WARN (AutoscalingActionExecutor-2911-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Exception executing actions
[junit4] 2> => org.apache.solr.cloud.autoscaling.TriggerActionException: Error processing action for trigger event: {
[junit4] 2> "id":"37ede5dafd8229Tf25b6mipzt86fio5b3ofk6amg",
[junit4] 2> org.apache.solr.cloud.autoscaling.TriggerActionException: Error processing action for trigger event: {
[junit4] 2> "id":"37ede5dafd8229Tf25b6mipzt86fio5b3ofk6amg",
[junit4] 2> "source":".auto_add_replicas",
[junit4] 2> "eventTime":15742695196426793,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[15742695196426793],
[junit4] 2> "__compute_diag__":{
[junit4] 2> "noSuggestionsStopAfter":2,
[junit4] 2> "estimatedMaxOps":12,
[junit4] 2> "requestedOperations":-1,
[junit4] 2> "opLimit":12,
[junit4] 2> "maxOperations":12},
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":15742749611684543,
[junit4] 2> "nodeNames":["127.0.0.1:10001_solr"]}}
[junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:334) ~[java/:?]
[junit4] 2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_281]
[junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_281]
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:218) ~[java/:?]
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_281]
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_281]
[junit4] 2> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_281]
[junit4] 2> Caused by: org.apache.solr.common.SolrException: Unexpected exception while processing event: {
[junit4] 2> "id":"37ede5dafd8229Tf25b6mipzt86fio5b3ofk6amg",
[junit4] 2> "source":".auto_add_replicas",
[junit4] 2> "eventTime":15742695196426793,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[15742695196426793],
[junit4] 2> "__compute_diag__":{
[junit4] 2> "noSuggestionsStopAfter":2,
[junit4] 2> "estimatedMaxOps":12,
[junit4] 2> "requestedOperations":-1,
[junit4] 2> "opLimit":12,
[junit4] 2> "maxOperations":12},
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":15742749611684543,
[junit4] 2> "nodeNames":["127.0.0.1:10001_solr"]}}
[junit4] 2> at org.apache.solr.cloud.autoscaling.ExecutePlanAction.process(ExecutePlanAction.java:188) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:331) ~[java/:?]
[junit4] 2> ... 6 more
[junit4] 2> Caused by: org.apache.solr.common.SolrException: Unexpected exception executing operation: action=MOVEREPLICA&async=.auto_add_replicas/37ede5dafd8229Tf25b6mipzt86fio5b3ofk6amg/0&waitForFinalState=true&collection=testCollection&targetNode=127.0.0.1:10000_solr&inPlaceMove=true&replica=core_node2
[junit4] 2> at org.apache.solr.cloud.autoscaling.ExecutePlanAction.process(ExecutePlanAction.java:178) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:331) ~[java/:?]
[junit4] 2> ... 6 more
[junit4] 2> Caused by: java.io.IOException: java.util.concurrent.ExecutionException: org.apache.solr.common.SolrException: java.lang.InterruptedException: sleep interrupted
[junit4] 2> at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.request(SimCloudManager.java:737) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.ExecutePlanAction.process(ExecutePlanAction.java:115) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:331) ~[java/:?]
[junit4] 2> ... 6 more
[junit4] 2> Caused by: java.util.concurrent.ExecutionException: org.apache.solr.common.SolrException: java.lang.InterruptedException: sleep interrupted
[junit4] 2> at java.util.concurrent.FutureTask.report(FutureTask.java:122) ~[?:1.8.0_281]
[junit4] 2> at java.util.concurrent.FutureTask.get(FutureTask.java:206) ~[?:1.8.0_281]
[junit4] 2> at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.request(SimCloudManager.java:735) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.ExecutePlanAction.process(ExecutePlanAction.java:115) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:331) ~[java/:?]
[junit4] 2> ... 6 more
[junit4] 2> Caused by: org.apache.solr.common.SolrException: java.lang.InterruptedException: sleep interrupted
[junit4] 2> at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.simHandleSolrRequest(SimCloudManager.java:930) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.lambda$request$15(SimCloudManager.java:734) ~[java/:?]
[junit4] 2> ... 5 more
[junit4] 2> Caused by: java.lang.InterruptedException: sleep interrupted
[junit4] 2> at java.lang.Thread.sleep(Native Method) ~[?:1.8.0_281]
[junit4] 2> at org.apache.solr.common.util.TimeSource$SimTimeSource.sleep(TimeSource.java:149) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.sim.SimClusterStateProvider.opDelay(SimClusterStateProvider.java:703) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.sim.SimClusterStateProvider.simMoveReplica(SimClusterStateProvider.java:1186) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.simHandleSolrRequest(SimCloudManager.java:928) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.lambda$request$15(SimCloudManager.java:734) ~[java/:?]
[junit4] 2> ... 5 more
[junit4] 2> 292615 DEBUG (AutoscalingActionExecutor-2911-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .scheduled_maintenance after 100ms
[junit4] 2> 292615 DEBUG (AutoscalingActionExecutor-2911-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms
[junit4] 2> 292615 DEBUG (AutoscalingActionExecutor-2911-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 5329 ms for event id=37ede5dafd8229Tf25b6mipzt86fio5b3ofk6amg
[junit4] 2> 292639 DEBUG (TEST-TestSimScenario.testAutoAddReplicas-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
[junit4] 2> 292639 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 292639 DEBUG (TEST-TestSimScenario.testAutoAddReplicas-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now
[junit4] 2> 292639 DEBUG (TEST-TestSimScenario.testAutoAddReplicas-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
[junit4] 2> 292641 DEBUG (TEST-TestSimScenario.testAutoAddReplicas-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor
[junit4] 2> 292641 DEBUG (TEST-TestSimScenario.testAutoAddReplicas-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
[junit4] 2> 292641 DEBUG (TEST-TestSimScenario.testAutoAddReplicas-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
[junit4] 2> 292643 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[7B42D9418824EC48]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testAutoAddReplicas
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestSimScenario -Dtests.method=testAutoAddReplicas -Dtests.seed=7B42D9418824EC48 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=hu -Dtests.timezone=CST6CDT -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] ERROR 1.45s J3 | TestSimScenario.testAutoAddReplicas <<<
[junit4] > Throwable #1: java.io.IOException: Timed out waiting for trigger .auto_add_replicas to fire after simulated 60s (real 1200ms).
[junit4] > at __randomizedtesting.SeedInfo.seed([7B42D9418824EC48:F2E22A3DFC2F2061]:0)
[junit4] > at org.apache.solr.cloud.autoscaling.sim.SimScenario$SimWaitListener.wait(SimScenario.java:705)
[junit4] > at org.apache.solr.cloud.autoscaling.sim.SimScenario$WaitEvent.execute(SimScenario.java:771)
[junit4] > at org.apache.solr.cloud.autoscaling.sim.SimScenario.run(SimScenario.java:1119)
[junit4] > at org.apache.solr.cloud.autoscaling.sim.TestSimScenario.testAutoAddReplicas(TestSimScenario.java:57)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 292653 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testSuggestions
[junit4] 2> 292655 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario 1. CreateCluster numNodes=2
[junit4] 2> 292655 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario CreateCluster numNodes=2
[junit4] 2> 292662 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimClusterStateProvider --- new Overseer leader: 127.0.0.1:10002_solr
[junit4] 2> 292662 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
[junit4] 2> 292662 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
[junit4] 2> 292663 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 0
[junit4] 2> 292663 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
[junit4] 2> 292671 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=0
[junit4] 2> 292672 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 0
[junit4] 2> 292672 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 292672 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario 2. RunSolrRequest path=/admin/collections&action=CREATE&autoAddReplicas=true&name=testCollection&numShards=2&replicationFactor=2&maxShardsPerNode=2
[junit4] 2> 292672 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario RunSolrRequest path=/admin/collections&action=CREATE&autoAddReplicas=true&name=testCollection&numShards=2&replicationFactor=2&maxShardsPerNode=2
[junit4] 2> 292672 DEBUG (simCloudManagerPool-2915-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider -- simCreateCollection testCollection, currentVersion=1
[junit4] 2> 292672 DEBUG (simCloudManagerPool-2915-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=1
[junit4] 2> 292672 DEBUG (simCloudManagerPool-2915-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 1
[junit4] 2> 292672 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10002_solr, 127.0.0.1:10003_solr]
[junit4] 2> 292673 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- deactivating old nodeLost / nodeAdded markers
[junit4] 2> 292673 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
[junit4] 2> 292676 DEBUG (ScheduledTrigger-2916-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 292683 DEBUG (simCloudManagerPool-2915-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=2
[junit4] 2> 292683 DEBUG (simCloudManagerPool-2915-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 2
[junit4] 2> 292701 DEBUG (ScheduledTrigger-2916-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 292710 DEBUG (simCloudManagerPool-2915-thread-2) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=3
[junit4] 2> 292711 DEBUG (simCloudManagerPool-2915-thread-2) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 3
[junit4] 2> 292711 DEBUG (simCloudManagerPool-2915-thread-2) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election (testCollection / shard1)
[junit4] 2> 292711 DEBUG (simCloudManagerPool-2915-thread-2) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testCollection / shard1 (currentVersion=4): {"core_node1":{}}
[junit4] 2> 292721 DEBUG (simCloudManagerPool-2915-thread-3) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=4
[junit4] 2> 292721 DEBUG (simCloudManagerPool-2915-thread-3) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 4
[junit4] 2> 292721 DEBUG (ScheduledTrigger-2916-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 292731 DEBUG (simCloudManagerPool-2915-thread-4) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=5
[junit4] 2> 292732 DEBUG (simCloudManagerPool-2915-thread-4) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 5
[junit4] 2> 292732 DEBUG (simCloudManagerPool-2915-thread-4) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election (testCollection / shard2)
[junit4] 2> 292732 DEBUG (simCloudManagerPool-2915-thread-4) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testCollection / shard2 (currentVersion=6): {"core_node3":{}}
[junit4] 2> 292732 DEBUG (simCloudManagerPool-2915-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider -- finished createCollection testCollection, currentVersion=6
[junit4] 2> 292732 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario 3. WaitCollection collection=testCollection&shards=2&replicas=2
[junit4] 2> 292732 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario WaitCollection collection=testCollection&shards=2&replicas=2
[junit4] 2> 292741 DEBUG (ScheduledTrigger-2916-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 292742 DEBUG (simCloudManagerPool-2915-thread-5) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=6
[junit4] 2> 292743 DEBUG (simCloudManagerPool-2915-thread-5) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 6
[junit4] 2> 292743 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=7
[junit4] 2> 292745 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 7
[junit4] 2> 292745 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario 4. CtxSet key=myNode&value=${_random_node_}
[junit4] 2> 292745 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario CtxSet key=myNode&value=127.0.0.1:10002_solr
[junit4] 2> 292746 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario 5. RunSolrRequest path=/admin/collections&action=ADDREPLICA&collection=testCollection&shard=shard1&node=${myNode}
[junit4] 2> 292746 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario RunSolrRequest path=/admin/collections&action=ADDREPLICA&collection=testCollection&shard=shard1&node=127.0.0.1:10002_solr
[junit4] 2> 292757 INFO (simCloudManagerPool-2915-thread-6) [ ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:10002_solr for creating new replica of shard shard1 for collection testCollection
[junit4] 2> 292757 INFO (simCloudManagerPool-2915-thread-6) [ ] o.a.s.c.a.c.AddReplicaCmd Returning CreateReplica command.
[junit4] 2> 292762 DEBUG (ScheduledTrigger-2916-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 292767 DEBUG (simCloudManagerPool-2915-thread-6) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=8
[junit4] 2> 292768 DEBUG (simCloudManagerPool-2915-thread-6) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 8
[junit4] 2> 292768 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario 6. RunSolrRequest path=/admin/collections&action=ADDREPLICA&collection=testCollection&shard=shard1&node=${myNode}
[junit4] 2> 292768 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario RunSolrRequest path=/admin/collections&action=ADDREPLICA&collection=testCollection&shard=shard1&node=127.0.0.1:10002_solr
[junit4] 2> 292776 DEBUG (simCloudManagerPool-2915-thread-7) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=9
[junit4] 2> 292776 DEBUG (simCloudManagerPool-2915-thread-7) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 9
[junit4] 2> 292777 INFO (simCloudManagerPool-2915-thread-7) [ ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:10002_solr for creating new replica of shard shard1 for collection testCollection
[junit4] 2> 292777 INFO (simCloudManagerPool-2915-thread-7) [ ] o.a.s.c.a.c.AddReplicaCmd Returning CreateReplica command.
[junit4] 2> 292782 DEBUG (ScheduledTrigger-2916-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 292788 DEBUG (simCloudManagerPool-2915-thread-7) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=10
[junit4] 2> 292788 DEBUG (simCloudManagerPool-2915-thread-7) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 10
[junit4] 2> 292788 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario 7. LoopOp iterations=${iterative}
[junit4] 2> 292788 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario LoopOp iterations=0
[junit4] 2> 292788 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario 8. LoopOp iterations=${justCalc}
[junit4] 2> 292789 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario LoopOp iterations=1
[junit4] 2> 292789 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario * iter 1 :
[junit4] 2> 292789 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario - CalculateSuggestions )
[junit4] 2> 292790 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=11
[junit4] 2> 292790 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 11
[junit4] 2> 292802 DEBUG (ScheduledTrigger-2916-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 292803 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario - 1 suggestions
[junit4] 2> 292803 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario - SaveSnapshot path=/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-Tests-8.8/solr/build/solr-core/test/J3/temp/solr.cloud.autoscaling.sim.TestSimScenario_7B42D9418824EC48-001/tempDir-002/snapshot/0)
[junit4] 2> 292820 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SnapshotDistribStateManager - copying 22 resources from SimDistribStateManager
[junit4] 2> 292823 DEBUG (ScheduledTrigger-2916-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 292836 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario 9. Dump redact=true
[junit4] 2> 292836 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario Dump redact=true
[junit4] 2> 292836 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SnapshotDistribStateManager - copying 22 resources from SimDistribStateManager
[junit4] 2> 292841 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Interrupted
[junit4] 2> => java.lang.InterruptedException
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.reportInterruptAfterWait(AbstractQueuedSynchronizer.java:2014)
[junit4] 2> java.lang.InterruptedException: null
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.reportInterruptAfterWait(AbstractQueuedSynchronizer.java:2014) ~[?:1.8.0_281]
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2048) ~[?:1.8.0_281]
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:209) [java/:?]
[junit4] 2> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_281]
[junit4] 2> 292841 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
[junit4] 2> 292842 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now
[junit4] 2> 292842 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
[junit4] 2> 292842 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor
[junit4] 2> 292842 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
[junit4] 2> 292842 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
[junit4] 2> 292842 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario 1. CreateCluster numNodes=2
[junit4] 2> 292842 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario CreateCluster numNodes=2
[junit4] 2> 292845 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimClusterStateProvider --- new Overseer leader: 127.0.0.1:10004_solr
[junit4] 2> 292847 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
[junit4] 2> 292847 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
[junit4] 2> 292848 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 0
[junit4] 2> 292848 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
[junit4] 2> 292850 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=0
[junit4] 2> 292850 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 0
[junit4] 2> 292850 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 292850 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario 2. RunSolrRequest path=/admin/collections&action=CREATE&autoAddReplicas=true&name=testCollection&numShards=2&replicationFactor=2&maxShardsPerNode=2
[junit4] 2> 292850 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario RunSolrRequest path=/admin/collections&action=CREATE&autoAddReplicas=true&name=testCollection&numShards=2&replicationFactor=2&maxShardsPerNode=2
[junit4] 2> 292852 DEBUG (simCloudManagerPool-2921-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider -- simCreateCollection testCollection, currentVersion=1
[junit4] 2> 292852 DEBUG (simCloudManagerPool-2921-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=1
[junit4] 2> 292852 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10004_solr, 127.0.0.1:10005_solr]
[junit4] 2> 292852 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- deactivating old nodeLost / nodeAdded markers
[junit4] 2> 292852 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
[junit4] 2> 292859 DEBUG (simCloudManagerPool-2921-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 1
[junit4] 2> 292859 DEBUG (ScheduledTrigger-2922-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 292869 DEBUG (simCloudManagerPool-2921-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=2
[junit4] 2> 292869 DEBUG (simCloudManagerPool-2921-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 2
[junit4] 2> 292887 DEBUG (ScheduledTrigger-2922-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 292892 DEBUG (simCloudManagerPool-2921-thread-2) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=3
[junit4] 2> 292892 DEBUG (simCloudManagerPool-2921-thread-2) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 3
[junit4] 2> 292892 DEBUG (simCloudManagerPool-2921-thread-2) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election (testCollection / shard1)
[junit4] 2> 292892 DEBUG (simCloudManagerPool-2921-thread-2) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testCollection / shard1 (currentVersion=4): {"core_node1":{}}
[junit4] 2> 292904 DEBUG (simCloudManagerPool-2921-thread-3) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=4
[junit4] 2> 292905 DEBUG (simCloudManagerPool-2921-thread-3) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 4
[junit4] 2> 292907 DEBUG (ScheduledTrigger-2922-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 292916 DEBUG (simCloudManagerPool-2921-thread-4) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=5
[junit4] 2> 292916 DEBUG (simCloudManagerPool-2921-thread-4) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 5
[junit4] 2> 292916 DEBUG (simCloudManagerPool-2921-thread-4) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election (testCollection / shard2)
[junit4] 2> 292916 DEBUG (simCloudManagerPool-2921-thread-4) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testCollection / shard2 (currentVersion=6): {"core_node3":{}}
[junit4] 2> 292916 DEBUG (simCloudManagerPool-2921-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider -- finished createCollection testCollection, currentVersion=6
[junit4] 2> 292916 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario 3. WaitCollection collection=testCollection&shards=2&replicas=2
[junit4] 2> 292916 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario WaitCollection collection=testCollection&shards=2&replicas=2
[junit4] 2> 292926 DEBUG (simCloudManagerPool-2921-thread-5) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=6
[junit4] 2> 292930 DEBUG (ScheduledTrigger-2922-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 292931 DEBUG (simCloudManagerPool-2921-thread-5) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 6
[junit4] 2> 292931 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=7
[junit4] 2> 292931 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 7
[junit4] 2> 292931 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario 4. CtxSet key=myNode&value=${_random_node_}
[junit4] 2> 292931 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario CtxSet key=myNode&value=127.0.0.1:10004_solr
[junit4] 2> 292931 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario 5. RunSolrRequest path=/admin/collections&action=ADDREPLICA&collection=testCollection&shard=shard1&node=${myNode}
[junit4] 2> 292931 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario RunSolrRequest path=/admin/collections&action=ADDREPLICA&collection=testCollection&shard=shard1&node=127.0.0.1:10004_solr
[junit4] 2> 292939 INFO (simCloudManagerPool-2921-thread-6) [ ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:10004_solr for creating new replica of shard shard1 for collection testCollection
[junit4] 2> 292939 INFO (simCloudManagerPool-2921-thread-6) [ ] o.a.s.c.a.c.AddReplicaCmd Returning CreateReplica command.
[junit4] 2> 292949 DEBUG (simCloudManagerPool-2921-thread-6) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=8
[junit4] 2> 292949 DEBUG (simCloudManagerPool-2921-thread-6) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 8
[junit4] 2> 292949 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario 6. RunSolrRequest path=/admin/collections&action=ADDREPLICA&collection=testCollection&shard=shard1&node=${myNode}
[junit4] 2> 292949 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario RunSolrRequest path=/admin/collections&action=ADDREPLICA&collection=testCollection&shard=shard1&node=127.0.0.1:10004_solr
[junit4] 2> 292950 DEBUG (simCloudManagerPool-2921-thread-7) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=9
[junit4] 2> 292950 DEBUG (ScheduledTrigger-2922-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 292950 DEBUG (simCloudManagerPool-2921-thread-7) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 9
[junit4] 2> 292950 INFO (simCloudManagerPool-2921-thread-7) [ ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:10004_solr for creating new replica of shard shard1 for collection testCollection
[junit4] 2> 292950 INFO (simCloudManagerPool-2921-thread-7) [ ] o.a.s.c.a.c.AddReplicaCmd Returning CreateReplica command.
[junit4] 2> 292961 DEBUG (simCloudManagerPool-2921-thread-7) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=10
[junit4] 2> 292961 DEBUG (simCloudManagerPool-2921-thread-7) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 10
[junit4] 2> 292961 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario 7. LoopOp iterations=${iterative}
[junit4] 2> 292961 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario LoopOp iterations=10
[junit4] 2> 292961 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario * iter 1 :
[junit4] 2> 292961 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario - CalculateSuggestions )
[junit4] 2> 292961 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=11
[junit4] 2> 292961 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 11
[junit4] 2> 292962 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario - 1 suggestions
[junit4] 2> 292962 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario - ApplySuggestions )
[junit4] 2> 292970 DEBUG (ScheduledTrigger-2922-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 292991 DEBUG (ScheduledTrigger-2922-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 293012 DEBUG (ScheduledTrigger-2922-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 293032 DEBUG (ScheduledTrigger-2922-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 293052 DEBUG (ScheduledTrigger-2922-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 293071 DEBUG (simCloudManagerPool-2921-thread-8) [ ] o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node10":{}}
[junit4] 2> 293072 DEBUG (ScheduledTrigger-2922-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 293093 DEBUG (ScheduledTrigger-2922-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 293116 DEBUG (ScheduledTrigger-2922-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 293136 DEBUG (ScheduledTrigger-2922-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 293156 DEBUG (ScheduledTrigger-2922-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 293177 DEBUG (ScheduledTrigger-2922-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 293181 DEBUG (simCloudManagerPool-2921-thread-8) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=12
[junit4] 2> 293182 DEBUG (simCloudManagerPool-2921-thread-8) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 12
[junit4] 2> 293182 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario - RunSolrRequest path=/admin/collections&action=ADDREPLICA&collection=testCollection&shard=shard1&node=127.0.0.1:10004_solr)
[junit4] 2> 293190 DEBUG (simCloudManagerPool-2921-thread-9) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=13
[junit4] 2> 293191 DEBUG (simCloudManagerPool-2921-thread-9) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 13
[junit4] 2> 293191 INFO (simCloudManagerPool-2921-thread-9) [ ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:10004_solr for creating new replica of shard shard1 for collection testCollection
[junit4] 2> 293191 INFO (simCloudManagerPool-2921-thread-9) [ ] o.a.s.c.a.c.AddReplicaCmd Returning CreateReplica command.
[junit4] 2> 293197 DEBUG (ScheduledTrigger-2922-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 293201 DEBUG (simCloudManagerPool-2921-thread-9) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=14
[junit4] 2> 293201 DEBUG (simCloudManagerPool-2921-thread-9) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 14
[junit4] 2> 293201 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario - RunSolrRequest path=/admin/collections&action=ADDREPLICA&collection=testCollection&shard=shard1&node=127.0.0.1:10004_solr)
[junit4] 2> 293202 DEBUG (simCloudManagerPool-2921-thread-10) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=15
[junit4] 2> 293203 DEBUG (simCloudManagerPool-2921-thread-10) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 15
[junit4] 2> 293204 INFO (simCloudManagerPool-2921-thread-10) [ ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:10004_solr for creating new replica of shard shard1 for collection testCollection
[junit4] 2> 293204 INFO (simCloudManagerPool-2921-thread-10) [ ] o.a.s.c.a.c.AddReplicaCmd Returning CreateReplica command.
[junit4] 2> 293215 DEBUG (simCloudManagerPool-2921-thread-10) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=16
[junit4] 2> 293217 DEBUG (simCloudManagerPool-2921-thread-10) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 16
[junit4] 2> 293217 DEBUG (ScheduledTrigger-2922-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 293220 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario * iter 2 :
[junit4] 2> 293220 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario - CalculateSuggestions )
[junit4] 2> 293220 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=17
[junit4] 2> 293220 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 17
[junit4] 2> 293221 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario - 1 suggestions
[junit4] 2> 293221 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario - ApplySuggestions )
[junit4] 2> 293240 DEBUG (ScheduledTrigger-2922-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 293260 DEBUG (ScheduledTrigger-2922-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 293280 DEBUG (ScheduledTrigger-2922-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 293300 DEBUG (ScheduledTrigger-2922-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 293321 DEBUG (ScheduledTrigger-2922-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 293323 DEBUG (simCloudManagerPool-2921-thread-11) [ ] o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node16":{}}
[junit4] 2> 293341 DEBUG (ScheduledTrigger-2922-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 293361 DEBUG (ScheduledTrigger-2922-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 293382 DEBUG (ScheduledTrigger-2922-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 293402 DEBUG (ScheduledTrigger-2922-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 293422 DEBUG (ScheduledTrigger-2922-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 293434 DEBUG (simCloudManagerPool-2921-thread-11) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=18
[junit4] 2> 293434 DEBUG (simCloudManagerPool-2921-thread-11) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 18
[junit4] 2> 293434 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario - RunSolrRequest path=/admin/collections&action=ADDREPLICA&collection=testCollection&shard=shard1&node=127.0.0.1:10004_solr)
[junit4] 2> 293434 DEBUG (simCloudManagerPool-2921-thread-12) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=19
[junit4] 2> 293436 DEBUG (simCloudManagerPool-2921-thread-12) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 19
[junit4] 2> 293437 INFO (simCloudManagerPool-2921-thread-12) [ ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:10004_solr for creating new replica of shard shard1 for collection testCollection
[junit4] 2> 293437 INFO (simCloudManagerPool-2921-thread-12) [ ] o.a.s.c.a.c.AddReplicaCmd Returning CreateReplica command.
[junit4] 2> 293442 DEBUG (ScheduledTrigger-2922-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 293447 DEBUG (simCloudManagerPool-2921-thread-12) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=20
[junit4] 2> 293448 DEBUG (simCloudManagerPool-2921-thread-12) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 20
[junit4] 2> 293448 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario - RunSolrRequest path=/admin/collections&action=ADDREPLICA&collection=testCollection&shard=shard1&node=127.0.0.1:10004_solr)
[junit4] 2> 293449 DEBUG (simCloudManagerPool-2921-thread-13) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=21
[junit4] 2> 293450 DEBUG (simCloudManagerPool-2921-thread-13) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 21
[junit4] 2> 293450 INFO (simCloudManagerPool-2921-thread-13) [ ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:10004_solr for creating new replica of shard shard1 for collection testCollection
[junit4] 2> 293450 INFO (simCloudManagerPool-2921-thread-13) [ ] o.a.s.c.a.c.AddReplicaCmd Returning CreateReplica command.
[junit4] 2> 293461 DEBUG (simCloudManagerPool-2921-thread-13) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=22
[junit4] 2> 293461 DEBUG (simCloudManagerPool-2921-thread-13) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 22
[junit4] 2> 293461 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario * iter 3 :
[junit4] 2> 293461 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario - CalculateSuggestions )
[junit4] 2> 293461 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=23
[junit4] 2> 293462 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 23
[junit4] 2> 293462 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario - 1 suggestions
[junit4] 2> 293462 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario - ApplySuggestions )
[junit4] 2> 293462 DEBUG (ScheduledTrigger-2922-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 293482 DEBUG (ScheduledTrigger-2922-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 293503 DEBUG (ScheduledTrigger-2922-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 293523 DEBUG (ScheduledTrigger-2922-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 293543 DEBUG (ScheduledTrigger-2922-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 293563 DEBUG (ScheduledTrigger-2922-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 293566 DEBUG (simCloudManagerPool-2921-thread-14) [ ] o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node22":{}}
[junit4] 2> 293583 DEBUG (ScheduledTrigger-2922-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 293604 DEBUG (ScheduledTrigger-2922-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 293624 DEBUG (ScheduledTrigger-2922-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 293644 DEBUG (ScheduledTrigger-2922-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 293664 DEBUG (ScheduledTrigger-2922-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 293678 DEBUG (simCloudManagerPool-2921-thread-14) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=24
[junit4] 2> 293678 DEBUG (simCloudManagerPool-2921-thread-14) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 24
[junit4] 2> 293678 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario - RunSolrRequest path=/admin/collections&action=ADDREPLICA&collection=testCollection&shard=shard1&node=127.0.0.1:10004_solr)
[junit4] 2> 293679 DEBUG (simCloudManagerPool-2921-thread-15) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=25
[junit4] 2> 293680 DEBUG (simCloudManagerPool-2921-thread-15) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 25
[junit4] 2> 293681 INFO (simCloudManagerPool-2921-thread-15) [ ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:10004_solr for creating new replica of shard shard1 for collection testCollection
[junit4] 2> 293681 INFO (simCloudManagerPool-2921-thread-15) [ ] o.a.s.c.a.c.AddReplicaCmd Returning CreateReplica command.
[junit4] 2> 293684 DEBUG (ScheduledTrigger-2922-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 293691 DEBUG (simCloudManagerPool-2921-thread-15) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=26
[junit4] 2> 293692 DEBUG (simCloudManagerPool-2921-thread-15) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 26
[junit4] 2> 293693 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario - RunSolrRequest path=/admin/collections&action=ADDREPLICA&collection=testCollection&shard=shard1&node=127.0.0.1:10004_solr)
[junit4] 2> 293693 DEBUG (simCloudManagerPool-2921-thread-16) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=27
[junit4] 2> 293693 DEBUG (simCloudManagerPool-2921-thread-16) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 27
[junit4] 2> 293694 INFO (simCloudManagerPool-2921-thread-16) [ ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:10004_solr for creating new replica of shard shard1 for collection testCollection
[junit4] 2> 293694 INFO (simCloudManagerPool-2921-thread-16) [ ] o.a.s.c.a.c.AddReplicaCmd Returning CreateReplica command.
[junit4] 2> 293704 DEBUG (ScheduledTrigger-2922-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 293704 DEBUG (simCloudManagerPool-2921-thread-16) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=28
[junit4] 2> 293705 DEBUG (simCloudManagerPool-2921-thread-16) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 28
[junit4] 2> 293705 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario * iter 4 :
[junit4] 2> 293705 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario - CalculateSuggestions )
[junit4] 2> 293705 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=29
[junit4] 2> 293705 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 29
[junit4] 2> 293707 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario - 1 suggestions
[junit4] 2> 293707 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario - ApplySuggestions )
[junit4] 2> 293725 DEBUG (ScheduledTrigger-2922-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 293745 DEBUG (ScheduledTrigger-2922-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 293766 DEBUG (ScheduledTrigger-2922-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 293786 DEBUG (ScheduledTrigger-2922-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 293806 DEBUG (ScheduledTrigger-2922-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 293809 DEBUG (simCloudManagerPool-2921-thread-17) [ ] o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node28":{}}
[junit4] 2> 293826 DEBUG (ScheduledTrigger-2922-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 293847 DEBUG (ScheduledTrigger-2922-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 293867 DEBUG (ScheduledTrigger-2922-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 293887 DEBUG (ScheduledTrigger-2922-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 293907 DEBUG (ScheduledTrigger-2922-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 293920 DEBUG (simCloudManagerPool-2921-thread-17) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=30
[junit4] 2> 293920 DEBUG (simCloudManagerPool-2921-thread-17) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 30
[junit4] 2> 293920 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario - RunSolrRequest path=/admin/collections&action=ADDREPLICA&collection=testCollection&shard=shard1&node=127.0.0.1:10004_solr)
[junit4] 2> 293921 DEBUG (simCloudManagerPool-2921-thread-18) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=31
[junit4] 2> 293921 DEBUG (simCloudManagerPool-2921-thread-18) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 31
[junit4] 2> 293922 INFO (simCloudManagerPool-2921-thread-18) [ ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:10004_solr for creating new replica of shard shard1 for collection testCollection
[junit4] 2> 293922 INFO (simCloudManagerPool-2921-thread-18) [ ] o.a.s.c.a.c.AddReplicaCmd Returning CreateReplica command.
[junit4] 2> 293927 DEBUG (ScheduledTrigger-2922-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 293932 DEBUG (simCloudManagerPool-2921-thread-18) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=32
[junit4] 2> 293932 DEBUG (simCloudManagerPool-2921-thread-18) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 32
[junit4] 2> 293932 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario - RunSolrRequest path=/admin/collections&action=ADDREPLICA&collection=testCollection&shard=shard1&node=127.0.0.1:10004_solr)
[junit4] 2> 293932 DEBUG (simCloudManagerPool-2921-thread-19) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=33
[junit4] 2> 293933 DEBUG (simCloudManagerPool-2921-thread-19) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 33
[junit4] 2> 293933 INFO (simCloudManagerPool-2921-thread-19) [ ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:10004_solr for creating new replica of shard shard1 for collection testCollection
[junit4] 2> 293934 INFO (simCloudManagerPool-2921-thread-19) [ ] o.a.s.c.a.c.AddReplicaCmd Returning CreateReplica command.
[junit4] 2> 293944 DEBUG (simCloudManagerPool-2921-thread-19) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=34
[junit4] 2> 293944 DEBUG (simCloudManagerPool-2921-thread-19) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 34
[junit4] 2> 293944 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario * iter 5 :
[junit4] 2> 293944 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario - CalculateSuggestions )
[junit4] 2> 293944 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=35
[junit4] 2> 293945 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 35
[junit4] 2> 293945 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario - 1 suggestions
[junit4] 2> 293945 INFO (TEST-TestSimScenario.testSuggestions-seed#[7B42D9418824EC48]) [ ] o.a.s.c.a.s.SimScenario - ApplySuggestions )
[junit4] 2> 293947 DEBUG (ScheduledTrigger-2922-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 293967 DEBUG (ScheduledTrigger-2922-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 293988 DEBUG (ScheduledTrigger-2922-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 294008 DEBUG (ScheduledTrigger-2922-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and las
[...truncated too long message...]
616ef
[junit4] 2> 2146968 INFO (searcherExecutor-21203-thread-1) [ ] o.a.s.c.SolrCore [collection1] Registered new searcher autowarm time: 0 ms
[junit4] 2> 2146975 INFO (TEST-SolrJmxReporterTest.testReloadCore-seed#[7B42D9418824EC48]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@48c4204a: rootName = SolrJmxReporterTest-wqjkhtfo, domain = solr.core.collection1, service url = null, agent id = localhost_1614698481715] for registry solr.core.collection1/com.codahale.metrics.MetricRegistry@263616ef
[junit4] 2> 2147103 INFO (TEST-SolrJmxReporterTest.testReloadCore-seed#[7B42D9418824EC48]) [ x:collection1 ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'SolrJmxReporterTest-wqjkhtfo.solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6601aacb
[junit4] 2> 2147111 INFO (TEST-SolrJmxReporterTest.testReloadCore-seed#[7B42D9418824EC48]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1 tag=null
[junit4] 2> 2147111 INFO (TEST-SolrJmxReporterTest.testReloadCore-seed#[7B42D9418824EC48]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@875f51e: rootName = null, domain = solr.core.collection1, service url = null, agent id = null] for registry solr.core.collection1/com.codahale.metrics.MetricRegistry@263616ef
[junit4] 2> 2147126 INFO (TEST-SolrJmxReporterTest.testReloadCore-seed#[7B42D9418824EC48]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@76ed400e: rootName = SolrJmxReporterTest-wqjkhtfo, domain = solr.core.collection1, service url = null, agent id = localhost_1614698481715] for registry solr.core.collection1/com.codahale.metrics.MetricRegistry@263616ef
[junit4] 2> 2147142 INFO (TEST-SolrJmxReporterTest.testReloadCore-seed#[7B42D9418824EC48]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1 tag=SolrCore@5d84a5a6
[junit4] 2> 2147142 INFO (TEST-SolrJmxReporterTest.testReloadCore-seed#[7B42D9418824EC48]) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> 2147142 INFO (TEST-SolrJmxReporterTest.testReloadCore-seed#[7B42D9418824EC48]) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=753368143
[junit4] 2> 2147142 INFO (coreCloseExecutor-21208-thread-1) [ ] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@342bc440
[junit4] 2> 2147142 INFO (coreCloseExecutor-21208-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1 tag=SolrCore@342bc440
[junit4] 2> 2147143 INFO (coreCloseExecutor-21208-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 Committing on IndexWriter.close() ... SKIPPED (unnecessary).
[junit4] 2> 2147143 INFO (TEST-SolrJmxReporterTest.testReloadCore-seed#[7B42D9418824EC48]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node tag=null
[junit4] 2> 2147143 INFO (TEST-SolrJmxReporterTest.testReloadCore-seed#[7B42D9418824EC48]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@5b3b1ebc: rootName = null, domain = solr.node, service url = null, agent id = null] for registry solr.node/com.codahale.metrics.MetricRegistry@2d25c744
[junit4] 2> 2147150 INFO (TEST-SolrJmxReporterTest.testReloadCore-seed#[7B42D9418824EC48]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm tag=null
[junit4] 2> 2147150 INFO (TEST-SolrJmxReporterTest.testReloadCore-seed#[7B42D9418824EC48]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@4c11e556: rootName = null, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm/com.codahale.metrics.MetricRegistry@77e5e343
[junit4] 2> 2147153 INFO (TEST-SolrJmxReporterTest.testReloadCore-seed#[7B42D9418824EC48]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty tag=null
[junit4] 2> 2147153 INFO (TEST-SolrJmxReporterTest.testReloadCore-seed#[7B42D9418824EC48]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@6516e722: rootName = null, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty/com.codahale.metrics.MetricRegistry@7f6c5237
[junit4] 2> 2147155 INFO (TEST-SolrJmxReporterTest.testReloadCore-seed#[7B42D9418824EC48]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testReloadCore
[junit4] 2> 2147156 INFO (SUITE-SolrJmxReporterTest-seed#[7B42D9418824EC48]-worker) [ ] o.a.s.SolrTestCaseJ4 ------------------------------------------------------- Done waiting for tracked resources to be released
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene87): {}, docValues:{}, maxPointsInLeafNode=617, maxMBSortInHeap=5.807371220396522, sim=Asserting(RandomSimilarity(queryNorm=false): {}), locale=es-PR, timezone=Navajo
[junit4] 2> NOTE: Linux 4.15.0-112-generic amd64/Oracle Corporation 1.8.0_281 (64-bit)/cpus=4,threads=1,free=222528592,total=529006592
[junit4] 2> NOTE: All tests run in this JVM: [TestZkAclsWithHadoopAuth, JSONWriterTest, ParsingFieldUpdateProcessorsTest, TestXmlQParserPlugin, AtomicUpdateProcessorFactoryTest, HdfsWriteToMultipleCollectionsTest, TestFieldResource, OverseerTest, DimensionalRoutedAliasUpdateProcessorTest, TestClassicSimilarityFactory, TestComponentsName, ExternalFileFieldSortTest, TestPayloadScoreQParserPlugin, TestFaceting, CheckHdfsIndexTest, PreAnalyzedFieldManagedSchemaCloudTest, DaemonStreamApiTest, OutputWriterTest, TestDocTermOrdsUninvertLimit, MoreLikeThisHandlerTest, TestObjectReleaseTracker, CloudMLTQParserTest, TestCircuitBreaker, NoCacheHeaderTest, TestRealTimeGet, TestCloudJSONFacetJoinDomain, BadComponentTest, TestFiltering, RequestHandlersTest, LeaderFailureAfterFreshStartTest, TestConfigSets, CoreSorterTest, TestExclusionRuleCollectionAccess, HdfsSyncSliceTest, MetricsHistoryIntegrationTest, TestMultiWordSynonyms, SolrMetricManagerTest, RulesTest, ShardRoutingCustomTest, SolrIndexSplitterTest, TestUnInvertedFieldException, TestSuggestSpellingConverter, MetricsHistoryHandlerTest, TestReloadDeadlock, ClusterStateMockUtilTest, ClassificationUpdateProcessorIntegrationTest, HdfsBasicDistributedZk2Test, RouteFieldTest, HdfsDirectoryFactoryTest, RankQParserPluginTest, CoreAdminHandlerTest, TestLockTree, NotRequiredUniqueKeyTest, TriggerCooldownIntegrationTest, TestSolrCoreParser, QueryElevationComponentTest, TimeZoneUtilsTest, TestSimExecutePlanAction, TestNumericRangeQuery32, TestDistribDocBasedVersion, TestSolrCloudSnapshots, TestCustomDocTransformer, TestSegmentSorting, DynamicMapsTest, CdcrRequestHandlerTest, CurrencyRangeFacetCloudTest, TestSolrJacksonAnnotation, TestSQLHandlerNonCloud, TestPerFieldSimilarity, DistributedVersionInfoTest, CircularListTest, XsltUpdateRequestHandlerTest, TestRTGBase, SolrTestCaseJ4Test, ReplaceNodeTest, TestSystemCollAutoCreate, TestTrackingShardHandlerFactory, DisMaxRequestHandlerTest, UpdateLogTest, TestJsonFacetRefinement, LoggingHandlerTest, RequiredFieldsTest, TestFieldCacheSortRandom, SolrCoreTest, TestDefaultStatsCache, TestSimNodeAddedTrigger, JvmMetricsTest, CollectionStateFormat2Test, SolrLogAuditLoggerPluginTest, HdfsNNFailoverTest, TestSimpleTextCodec, TestPHPSerializedResponseWriter, NodeAddedTriggerTest, TestInPlaceUpdateWithRouteField, DeleteNodeTest, XmlInterpolationTest, PathHierarchyTokenizerFactoryTest, CrossCollectionJoinQueryTest, SolrIndexSearcherTest, PrimUtilsTest, TestExtendedDismaxParser, TestCloudSearcherWarming, BJQParserTest, SpatialRPTFieldTypeTest, TestSchemaResource, TestQuerySenderListener, SuggestComponentTest, TestCaffeineCache, RegexBytesRefFilterTest, HighlighterTest, BasicZkTest, AliasIntegrationTest, TestConfigSetsAPIShareSchema, AnalysisAfterCoreReloadTest, BasicFunctionalityTest, ConvertedLegacyTest, SolrTestCaseJ4DeleteCoreTest, TestDistributedGrouping, TestDocumentBuilder, TestGroupingSearch, TestHighlightDedupGrouping, TestJoin, TestRandomFaceting, TestJettySolrRunner, ActionThrottleTest, AddReplicaTest, CleanupOldIndexTest, CollectionPropsTest, DeleteInactiveReplicaTest, DistribCursorPagingTest, FullSolrCloudDistribCmdsTest, LeaderElectionContextKeyTest, LeaderElectionIntegrationTest, LeaderElectionTest, MultiSolrCloudTestCaseTest, NestedShardedAtomicUpdateTest, NodeMutatorTest, OverriddenZkACLAndCredentialsProvidersTest, OverseerSolrResponseTest, OverseerSolrResponseUnsafeSerializationTest, OverseerStatusTest, ReindexCollectionTest, TestCloudRecovery, TestPullReplica, TestPullReplicaErrorHandling, TestSolrCloudWithSecureImpersonation, TestStressLiveNodes, HdfsCollectionsAPIDistributedZkTest, ShardSplitTest, IndexSizeTriggerTest, CdcrReplicationHandlerTest, HdfsUnloadDistributedZkTest, ZkStateWriterTest, ConfigureRecoveryStrategyTest, TestConfig, TestConfigOverlay, TestConfigSetImmutable, TestCustomStream, TestDynamicLoadingUrl, TestImplicitCoreProperties, TestInfoStreamLogging, TestJmxIntegration, TestMergePolicyConfig, TestSolrDeletionPolicy2, TestSolrXml, CSVRequestHandlerTest, DocumentAnalysisRequestHandlerTest, TestBlobHandler, TestReqParamsAPI, SplitHandlerTest, SystemInfoHandlerTest, TestCollectionAPIs, ThreadDumpHandlerTest, DebugComponentTest, DistributedExpandComponentTest, DistributedFacetPivotLongTailTest, DistributedQueryComponentCustomSortTest, DistributedSpellCheckComponentTest, InfixSuggestersTest, StatsComponentTest, TestPivotHelperCode, EmbeddedSolrNoSerializeTest, Tagger2Test, TaggerTest, HighlighterMaxOffsetTest, TestSlowCompositeReaderWrapper, UninvertDocValuesMergePolicyTest, TestLegacyNumericUtils, SolrJmxReporterTest]
[junit4] Completed [718/923 (2!)] on J0 in 10.21s, 4 tests, 1 error <<< FAILURES!
[...truncated 51491 lines...]