You are viewing a plain text version of this content. The canonical link for it is here.
Posted to builds@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2021/10/19 05:51:09 UTC
[JENKINS] Lucene » Lucene-Solr-Tests-8.10 - Build # 236 - Unstable!
Build: https://ci-builds.apache.org/job/Lucene/job/Lucene-Solr-Tests-8.10/236/
1 tests failed.
FAILED: org.apache.solr.cloud.autoscaling.sim.TestSimScenario.testAutoAddReplicas
Error Message:
Timed out waiting for trigger .auto_add_replicas to fire after simulated 60s (real 1200ms).
Stack Trace:
java.io.IOException: Timed out waiting for trigger .auto_add_replicas to fire after simulated 60s (real 1200ms).
at __randomizedtesting.SeedInfo.seed([93D020C624824834:1A70D3BA5089841D]:0)
at org.apache.solr.cloud.autoscaling.sim.SimScenario$SimWaitListener.wait(SimScenario.java:705)
at org.apache.solr.cloud.autoscaling.sim.SimScenario$WaitEvent.execute(SimScenario.java:771)
at org.apache.solr.cloud.autoscaling.sim.SimScenario.run(SimScenario.java:1119)
at org.apache.solr.cloud.autoscaling.sim.TestSimScenario.testAutoAddReplicas(TestSimScenario.java:57)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 14768 lines...]
[junit4] Suite: org.apache.solr.cloud.autoscaling.sim.TestSimScenario
[junit4] 2> 872631 INFO (SUITE-TestSimScenario-seed#[93D020C624824834]-worker) [ ] o.a.s.SolrTestCase Setting 'solr.default.confdir' system property to test-framework derived value of '/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-Tests-8.10/solr/server/solr/configsets/_default/conf'
[junit4] 2> 872631 INFO (SUITE-TestSimScenario-seed#[93D020C624824834]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 872632 INFO (SUITE-TestSimScenario-seed#[93D020C624824834]-worker) [ ] o.a.s.SolrTestCaseJ4 Created dataDir: /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-Tests-8.10/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.sim.TestSimScenario_93D020C624824834-001/data-dir-63-001
[junit4] 2> 872632 INFO (SUITE-TestSimScenario-seed#[93D020C624824834]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 872633 INFO (SUITE-TestSimScenario-seed#[93D020C624824834]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 872636 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[93D020C624824834]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testAutoAddReplicas
[junit4] 2> 872856 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario 1. CreateCluster numNodes=2
[junit4] 2> 872857 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario CreateCluster numNodes=2
[junit4] 2> 872866 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[93D020C624824834]) [ ] o.a.s.c.SolrPaths solr home defaulted to 'solr/' (could not find system property or JNDI)
[junit4] 2> 872891 DEBUG (TEST-TestSimScenario.testAutoAddReplicas-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimClusterStateProvider --- new Overseer leader: 127.0.0.1:10000_solr
[junit4] 2> 872891 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
[junit4] 2> 872892 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
[junit4] 2> 872892 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 0
[junit4] 2> 872892 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
[junit4] 2> 872894 DEBUG (TEST-TestSimScenario.testAutoAddReplicas-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=0
[junit4] 2> 872895 DEBUG (TEST-TestSimScenario.testAutoAddReplicas-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 0
[junit4] 2> 872895 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[93D020C624824834]) [ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 872895 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario 2. LoadAutoscaling json={'cluster-policy'+:+[{'replica'+:+'<3',+'shard'+:+'#EACH',+'collection'+:+'testCollection','node':'#ANY'}]}&defaultWaitFor=10
[junit4] 2> 872895 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario LoadAutoscaling json={'cluster-policy'+:+[{'replica'+:+'<3',+'shard'+:+'#EACH',+'collection'+:+'testCollection','node':'#ANY'}]}&defaultWaitFor=10
[junit4] 2> 872896 DEBUG (TEST-TestSimScenario.testAutoAddReplicas-seed#[93D020C624824834]) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2
[junit4] 2> 872901 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10000_solr]
[junit4] 2> 872910 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- deactivating old nodeLost / nodeAdded markers
[junit4] 2> 872910 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 1
[junit4] 2> 872910 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
[junit4] 2> 872910 DEBUG (ScheduledTrigger-7844-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 872912 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario 3. RunSolrRequest path=/admin/collections&action=CREATE&autoAddReplicas=true&name=testCollection&numShards=2&replicationFactor=2&maxShardsPerNode=2
[junit4] 2> 872912 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario RunSolrRequest path=/admin/collections&action=CREATE&autoAddReplicas=true&name=testCollection&numShards=2&replicationFactor=2&maxShardsPerNode=2
[junit4] 2> 872923 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10000_solr]
[junit4] 2> 872924 DEBUG (ScheduledTrigger-7844-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 872924 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- deactivating old nodeLost / nodeAdded markers
[junit4] 2> 872924 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
[junit4] 2> 872926 DEBUG (simCloudManagerPool-7843-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider -- simCreateCollection testCollection, currentVersion=1
[junit4] 2> 872926 DEBUG (simCloudManagerPool-7843-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=1
[junit4] 2> 872927 DEBUG (simCloudManagerPool-7843-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 1
[junit4] 2> 872939 DEBUG (simCloudManagerPool-7843-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=2
[junit4] 2> 872940 DEBUG (simCloudManagerPool-7843-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 2
[junit4] 2> 872945 DEBUG (ScheduledTrigger-7844-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 872965 DEBUG (ScheduledTrigger-7844-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 872989 DEBUG (simCloudManagerPool-7843-thread-2) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=3
[junit4] 2> 872989 DEBUG (simCloudManagerPool-7843-thread-2) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 3
[junit4] 2> 872989 DEBUG (simCloudManagerPool-7843-thread-2) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election (testCollection / shard1)
[junit4] 2> 872990 DEBUG (simCloudManagerPool-7843-thread-2) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testCollection / shard1 (currentVersion=4): {"core_node1":{}}
[junit4] 2> 872991 DEBUG (ScheduledTrigger-7844-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 873001 DEBUG (simCloudManagerPool-7843-thread-3) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=4
[junit4] 2> 873001 DEBUG (simCloudManagerPool-7843-thread-3) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 4
[junit4] 2> 873011 DEBUG (ScheduledTrigger-7844-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 873011 DEBUG (simCloudManagerPool-7843-thread-4) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=5
[junit4] 2> 873012 DEBUG (simCloudManagerPool-7843-thread-4) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 5
[junit4] 2> 873012 DEBUG (simCloudManagerPool-7843-thread-4) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election (testCollection / shard2)
[junit4] 2> 873012 DEBUG (simCloudManagerPool-7843-thread-4) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testCollection / shard2 (currentVersion=6): {"core_node3":{}}
[junit4] 2> 873013 DEBUG (simCloudManagerPool-7843-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider -- finished createCollection testCollection, currentVersion=6
[junit4] 2> 873013 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario 4. WaitCollection collection=testCollection&shards=2&replicas=2
[junit4] 2> 873013 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario WaitCollection collection=testCollection&shards=2&replicas=2
[junit4] 2> 873023 DEBUG (simCloudManagerPool-7843-thread-5) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=6
[junit4] 2> 873024 DEBUG (simCloudManagerPool-7843-thread-5) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 6
[junit4] 2> 873024 DEBUG (TEST-TestSimScenario.testAutoAddReplicas-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=7
[junit4] 2> 873024 DEBUG (TEST-TestSimScenario.testAutoAddReplicas-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 7
[junit4] 2> 873024 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario 5. SetEventListener trigger=.auto_add_replicas&stage=SUCCEEDED
[junit4] 2> 873024 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario SetEventListener trigger=.auto_add_replicas&stage=SUCCEEDED
[junit4] 2> 873024 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario 6. KillNodes node=${_random_node_}
[junit4] 2> 873025 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario KillNodes node=127.0.0.1:10000_solr
[junit4] 2> 873025 DEBUG (TEST-TestSimScenario.testAutoAddReplicas-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimClusterStateProvider -- created marker: /autoscaling/nodeLost/127.0.0.1:10000_solr
[junit4] 2> 873025 DEBUG (TEST-TestSimScenario.testAutoAddReplicas-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimClusterStateProvider --- new Overseer leader: 127.0.0.1:10001_solr
[junit4] 2> 873025 DEBUG (TEST-TestSimScenario.testAutoAddReplicas-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=8
[junit4] 2> 873025 DEBUG (TEST-TestSimScenario.testAutoAddReplicas-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 8
[junit4] 2> 873031 DEBUG (ScheduledTrigger-7844-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 2
[junit4] 2> 873031 DEBUG (ScheduledTrigger-7844-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10000_solr
[junit4] 2> 873048 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario 7. WaitEvent trigger=.auto_add_replicas&wait=60
[junit4] 2> 873048 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario WaitEvent trigger=.auto_add_replicas&wait=60
[junit4] 2> 873049 DEBUG (simCloudManagerPool-7843-thread-7) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election (testCollection / shard1)
[junit4] 2> 873049 INFO (simCloudManagerPool-7843-thread-7) [ ] o.a.s.c.ActionThrottle Throttling leader attempts - waiting for 2053ms
[junit4] 2> 873052 DEBUG (ScheduledTrigger-7844-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
[junit4] 2> 873072 DEBUG (ScheduledTrigger-7844-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
[junit4] 2> 873090 DEBUG (simCloudManagerPool-7843-thread-7) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testCollection / shard1 (currentVersion=9): {"core_node2":{}}
[junit4] 2> 873090 DEBUG (simCloudManagerPool-7843-thread-6) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=9
[junit4] 2> 873091 DEBUG (simCloudManagerPool-7843-thread-6) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 9
[junit4] 2> 873092 DEBUG (ScheduledTrigger-7844-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
[junit4] 2> 873112 DEBUG (ScheduledTrigger-7844-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
[junit4] 2> 873132 DEBUG (ScheduledTrigger-7844-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
[junit4] 2> 873152 DEBUG (ScheduledTrigger-7844-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
[junit4] 2> 873172 DEBUG (ScheduledTrigger-7844-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
[junit4] 2> 873192 DEBUG (ScheduledTrigger-7844-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
[junit4] 2> 873213 DEBUG (ScheduledTrigger-7844-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
[junit4] 2> 873233 DEBUG (ScheduledTrigger-7844-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
[junit4] 2> 873233 DEBUG (ScheduledTrigger-7844-thread-4) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:10000_solr]
[junit4] 2> 873233 DEBUG (ScheduledTrigger-7844-thread-4) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
[junit4] 2> "id":"7eb60a61544bdfTdf2eqzmzs4x9sz7htw5qagp23",
[junit4] 2> "source":".auto_add_replicas",
[junit4] 2> "eventTime":35666002764385247,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[35666002764385247],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "nodeNames":["127.0.0.1:10000_solr"]}}
[junit4] 2> 873233 DEBUG (ScheduledTrigger-7844-thread-4) [ ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, .scheduled_maintenance]
[junit4] 2> 873234 DEBUG (ScheduledTrigger-7844-thread-4) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=10
[junit4] 2> 873234 DEBUG (ScheduledTrigger-7844-thread-4) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 10
[junit4] 2> 873234 DEBUG (ScheduledTrigger-7844-thread-4) [ ] o.a.s.c.a.SystemLogListener Collection .system missing, skip sending event {
[junit4] 2> "id":"7eb60a61544bdfTdf2eqzmzs4x9sz7htw5qagp23",
[junit4] 2> "source":".auto_add_replicas",
[junit4] 2> "eventTime":35666002764385247,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[35666002764385247],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":35666012863884997,
[junit4] 2> "nodeNames":["127.0.0.1:10000_solr"]}}
[junit4] 2> 873244 DEBUG (AutoscalingActionExecutor-7845-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
[junit4] 2> "id":"7eb60a61544bdfTdf2eqzmzs4x9sz7htw5qagp23",
[junit4] 2> "source":".auto_add_replicas",
[junit4] 2> "eventTime":35666002764385247,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[35666002764385247],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":35666012863884997,
[junit4] 2> "nodeNames":["127.0.0.1:10000_solr"]}}
[junit4] 2> 873244 DEBUG (AutoscalingActionExecutor-7845-thread-1) [ ] o.a.s.c.a.ComputePlanAction -- processing event: {
[junit4] 2> "id":"7eb60a61544bdfTdf2eqzmzs4x9sz7htw5qagp23",
[junit4] 2> "source":".auto_add_replicas",
[junit4] 2> "eventTime":35666002764385247,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[35666002764385247],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":35666012863884997,
[junit4] 2> "nodeNames":["127.0.0.1:10000_solr"]}} with context properties: {BEFORE_ACTION=[auto_add_replicas_plan]}
[junit4] 2> 873247 DEBUG (AutoscalingActionExecutor-7845-thread-1) [ ] o.a.s.c.a.ComputePlanAction -- estimated total max ops=12, effective maxOps=12
[junit4] 2> 873248 DEBUG (AutoscalingActionExecutor-7845-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testCollection&targetNode=127.0.0.1:10001_solr&inPlaceMove=true&replica=core_node1
[junit4] 2> 873248 DEBUG (AutoscalingActionExecutor-7845-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testCollection&targetNode=127.0.0.1:10001_solr&inPlaceMove=true&replica=core_node4
[junit4] 2> 873249 DEBUG (AutoscalingActionExecutor-7845-thread-1) [ ] o.a.s.c.a.ComputePlanAction -- no more operations suggested, stopping after 2 ops...
[junit4] 2> 873249 DEBUG (AutoscalingActionExecutor-7845-thread-1) [ ] o.a.s.c.a.ExecutePlanAction -- processing event: {
[junit4] 2> "id":"7eb60a61544bdfTdf2eqzmzs4x9sz7htw5qagp23",
[junit4] 2> "source":".auto_add_replicas",
[junit4] 2> "eventTime":35666002764385247,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[35666002764385247],
[junit4] 2> "__compute_diag__":{
[junit4] 2> "noSuggestionsStopAfter":2,
[junit4] 2> "estimatedMaxOps":12,
[junit4] 2> "requestedOperations":-1,
[junit4] 2> "opLimit":12,
[junit4] 2> "maxOperations":12},
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":35666012863884997,
[junit4] 2> "nodeNames":["127.0.0.1:10000_solr"]}} with context properties: {operations=[{
[junit4] 2> "class":"org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica",
[junit4] 2> "method":"GET",
[junit4] 2> "params.action":"MOVEREPLICA",
[junit4] 2> "params.collection":"testCollection",
[junit4] 2> "params.targetNode":"127.0.0.1:10001_solr",
[junit4] 2> "params.inPlaceMove":"true",
[junit4] 2> "params.replica":"core_node1"}, {
[junit4] 2> "class":"org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica",
[junit4] 2> "method":"GET",
[junit4] 2> "params.action":"MOVEREPLICA",
[junit4] 2> "params.collection":"testCollection",
[junit4] 2> "params.targetNode":"127.0.0.1:10001_solr",
[junit4] 2> "params.inPlaceMove":"true",
[junit4] 2> "params.replica":"core_node4"}], AFTER_ACTION=[auto_add_replicas_plan], BEFORE_ACTION=[auto_add_replicas_plan, execute_plan]}
[junit4] 2> 873249 DEBUG (AutoscalingActionExecutor-7845-thread-1) [ ] o.a.s.c.a.ExecutePlanAction Executing operation: action=MOVEREPLICA&collection=testCollection&targetNode=127.0.0.1:10001_solr&inPlaceMove=true&replica=core_node1
[junit4] 2> 873584 DEBUG (simCloudManagerPool-7843-thread-8) [ ] o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node6":{}}
[junit4] 2> 873814 DEBUG (simCloudManagerPool-7843-thread-8) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=11
[junit4] 2> 873815 DEBUG (simCloudManagerPool-7843-thread-8) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 11
[junit4] 2> 873913 DEBUG (AutoscalingActionExecutor-7845-thread-1) [ ] o.a.s.c.a.ExecutePlanAction Executing operation: action=MOVEREPLICA&collection=testCollection&targetNode=127.0.0.1:10001_solr&inPlaceMove=true&replica=core_node4
[junit4] 2> 873913 DEBUG (simCloudManagerPool-7843-thread-11) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=12
[junit4] 2> 873914 DEBUG (simCloudManagerPool-7843-thread-11) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 12
[junit4] 2> 874014 DEBUG (simCloudManagerPool-7843-thread-11) [ ] o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node8":{}}
[junit4] 2> 874303 DEBUG (simCloudManagerPool-7843-thread-11) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=13
[junit4] 2> 874303 DEBUG (simCloudManagerPool-7843-thread-11) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 13
[junit4] 2> 874305 DEBUG (TEST-TestSimScenario.testAutoAddReplicas-seed#[93D020C624824834]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
[junit4] 2> 874305 DEBUG (TEST-TestSimScenario.testAutoAddReplicas-seed#[93D020C624824834]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now
[junit4] 2> 874305 DEBUG (TEST-TestSimScenario.testAutoAddReplicas-seed#[93D020C624824834]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
[junit4] 2> 874305 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 874312 ERROR (AutoscalingActionExecutor-7845-thread-1) [ ] o.a.s.c.a.ExecutePlanAction Unexpected Exception while querying status of requestId=.auto_add_replicas/7eb60a61544bdfTdf2eqzmzs4x9sz7htw5qagp23/1
[junit4] 2> => java.io.IOException: java.util.concurrent.RejectedExecutionException: Task org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$61/812920673@729755df rejected from org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor@60e9c170[Shutting down, pool size = 4, active threads = 0, queued tasks = 0, completed tasks = 11]
[junit4] 2> at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.request(SimCloudManager.java:737)
[junit4] 2> java.io.IOException: java.util.concurrent.RejectedExecutionException: Task org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$61/812920673@729755df rejected from org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor@60e9c170[Shutting down, pool size = 4, active threads = 0, queued tasks = 0, completed tasks = 11]
[junit4] 2> at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.request(SimCloudManager.java:737) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.ExecutePlanAction.waitForTaskToFinish(ExecutePlanAction.java:200) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.ExecutePlanAction.process(ExecutePlanAction.java:120) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:331) ~[java/:?]
[junit4] 2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_291]
[junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_291]
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:218) ~[java/:?]
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_291]
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_291]
[junit4] 2> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_291]
[junit4] 2> Caused by: java.util.concurrent.RejectedExecutionException: Task org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$61/812920673@729755df rejected from org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor@60e9c170[Shutting down, pool size = 4, active threads = 0, queued tasks = 0, completed tasks = 11]
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2063) ~[?:1.8.0_291]
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:830) ~[?:1.8.0_291]
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1379) ~[?:1.8.0_291]
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.execute(ExecutorUtil.java:203) ~[java/:?]
[junit4] 2> at java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:134) ~[?:1.8.0_291]
[junit4] 2> at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.request(SimCloudManager.java:734) ~[java/:?]
[junit4] 2> ... 9 more
[junit4] 2> 874312 WARN (AutoscalingActionExecutor-7845-thread-1) [ ] o.a.s.c.a.SystemLogListener Exception sending event. Collection: .system, event: {
[junit4] 2> "id":"7eb60a61544bdfTdf2eqzmzs4x9sz7htw5qagp23",
[junit4] 2> "source":".auto_add_replicas",
[junit4] 2> "eventTime":35666002764385247,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[35666002764385247],
[junit4] 2> "__compute_diag__":{
[junit4] 2> "noSuggestionsStopAfter":2,
[junit4] 2> "estimatedMaxOps":12,
[junit4] 2> "requestedOperations":-1,
[junit4] 2> "opLimit":12,
[junit4] 2> "maxOperations":12},
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":35666012863884997,
[junit4] 2> "nodeNames":["127.0.0.1:10000_solr"]}}, exception: java.io.IOException: already closed
[junit4] 2> 874312 WARN (AutoscalingActionExecutor-7845-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Exception executing actions
[junit4] 2> => org.apache.solr.cloud.autoscaling.TriggerActionException: Error processing action for trigger event: {
[junit4] 2> "id":"7eb60a61544bdfTdf2eqzmzs4x9sz7htw5qagp23",
[junit4] 2> org.apache.solr.cloud.autoscaling.TriggerActionException: Error processing action for trigger event: {
[junit4] 2> "id":"7eb60a61544bdfTdf2eqzmzs4x9sz7htw5qagp23",
[junit4] 2> "source":".auto_add_replicas",
[junit4] 2> "eventTime":35666002764385247,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[35666002764385247],
[junit4] 2> "__compute_diag__":{
[junit4] 2> "noSuggestionsStopAfter":2,
[junit4] 2> "estimatedMaxOps":12,
[junit4] 2> "requestedOperations":-1,
[junit4] 2> "opLimit":12,
[junit4] 2> "maxOperations":12},
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":35666012863884997,
[junit4] 2> "nodeNames":["127.0.0.1:10000_solr"]}}
[junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:334) ~[java/:?]
[junit4] 2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_291]
[junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_291]
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:218) ~[java/:?]
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_291]
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_291]
[junit4] 2> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_291]
[junit4] 2> Caused by: org.apache.solr.common.SolrException: Unexpected exception while processing event: {
[junit4] 2> "id":"7eb60a61544bdfTdf2eqzmzs4x9sz7htw5qagp23",
[junit4] 2> "source":".auto_add_replicas",
[junit4] 2> "eventTime":35666002764385247,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[35666002764385247],
[junit4] 2> "__compute_diag__":{
[junit4] 2> "noSuggestionsStopAfter":2,
[junit4] 2> "estimatedMaxOps":12,
[junit4] 2> "requestedOperations":-1,
[junit4] 2> "opLimit":12,
[junit4] 2> "maxOperations":12},
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":35666012863884997,
[junit4] 2> "nodeNames":["127.0.0.1:10000_solr"]}}
[junit4] 2> at org.apache.solr.cloud.autoscaling.ExecutePlanAction.process(ExecutePlanAction.java:188) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:331) ~[java/:?]
[junit4] 2> ... 6 more
[junit4] 2> Caused by: org.apache.solr.common.SolrException: Unexpected exception executing operation: action=MOVEREPLICA&async=.auto_add_replicas/7eb60a61544bdfTdf2eqzmzs4x9sz7htw5qagp23/1&waitForFinalState=true&collection=testCollection&targetNode=127.0.0.1:10001_solr&inPlaceMove=true&replica=core_node4
[junit4] 2> at org.apache.solr.cloud.autoscaling.ExecutePlanAction.process(ExecutePlanAction.java:178) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:331) ~[java/:?]
[junit4] 2> ... 6 more
[junit4] 2> Caused by: java.io.IOException: java.util.concurrent.RejectedExecutionException: Task org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$61/812920673@729755df rejected from org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor@60e9c170[Shutting down, pool size = 4, active threads = 0, queued tasks = 0, completed tasks = 11]
[junit4] 2> at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.request(SimCloudManager.java:737) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.ExecutePlanAction.waitForTaskToFinish(ExecutePlanAction.java:200) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.ExecutePlanAction.process(ExecutePlanAction.java:120) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:331) ~[java/:?]
[junit4] 2> ... 6 more
[junit4] 2> Caused by: java.util.concurrent.RejectedExecutionException: Task org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$61/812920673@729755df rejected from org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor@60e9c170[Shutting down, pool size = 4, active threads = 0, queued tasks = 0, completed tasks = 11]
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2063) ~[?:1.8.0_291]
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:830) ~[?:1.8.0_291]
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1379) ~[?:1.8.0_291]
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.execute(ExecutorUtil.java:203) ~[java/:?]
[junit4] 2> at java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:134) ~[?:1.8.0_291]
[junit4] 2> at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.request(SimCloudManager.java:734) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.ExecutePlanAction.waitForTaskToFinish(ExecutePlanAction.java:200) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.ExecutePlanAction.process(ExecutePlanAction.java:120) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:331) ~[java/:?]
[junit4] 2> ... 6 more
[junit4] 2> 874313 DEBUG (AutoscalingActionExecutor-7845-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 53453 ms for event id=7eb60a61544bdfTdf2eqzmzs4x9sz7htw5qagp23
[junit4] 2> 874313 DEBUG (TEST-TestSimScenario.testAutoAddReplicas-seed#[93D020C624824834]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor
[junit4] 2> 874313 DEBUG (TEST-TestSimScenario.testAutoAddReplicas-seed#[93D020C624824834]) [ ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
[junit4] 2> 874313 DEBUG (TEST-TestSimScenario.testAutoAddReplicas-seed#[93D020C624824834]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
[junit4] 2> 874314 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[93D020C624824834]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testAutoAddReplicas
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestSimScenario -Dtests.method=testAutoAddReplicas -Dtests.seed=93D020C624824834 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=et -Dtests.timezone=America/Indiana/Vincennes -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
[junit4] ERROR 1.74s J0 | TestSimScenario.testAutoAddReplicas <<<
[junit4] > Throwable #1: java.io.IOException: Timed out waiting for trigger .auto_add_replicas to fire after simulated 60s (real 1200ms).
[junit4] > at __randomizedtesting.SeedInfo.seed([93D020C624824834:1A70D3BA5089841D]:0)
[junit4] > at org.apache.solr.cloud.autoscaling.sim.SimScenario$SimWaitListener.wait(SimScenario.java:705)
[junit4] > at org.apache.solr.cloud.autoscaling.sim.SimScenario$WaitEvent.execute(SimScenario.java:771)
[junit4] > at org.apache.solr.cloud.autoscaling.sim.SimScenario.run(SimScenario.java:1119)
[junit4] > at org.apache.solr.cloud.autoscaling.sim.TestSimScenario.testAutoAddReplicas(TestSimScenario.java:57)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 874373 INFO (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testSplitShard
[junit4] 2> 874373 INFO (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario 1. CreateCluster numNodes=2
[junit4] 2> 874373 INFO (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario CreateCluster numNodes=2
[junit4] 2> 874383 DEBUG (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimClusterStateProvider --- new Overseer leader: 127.0.0.1:10002_solr
[junit4] 2> 874384 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
[junit4] 2> 874384 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
[junit4] 2> 874384 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 0
[junit4] 2> 874384 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
[junit4] 2> 874403 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10002_solr]
[junit4] 2> 874409 DEBUG (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=0
[junit4] 2> 874409 DEBUG (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 0
[junit4] 2> 874409 INFO (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 874409 INFO (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario 2. RunSolrRequest path=/admin/collections&action=CREATE&name=testCollection&numShards=2&replicationFactor=2&maxShardsPerNode=5
[junit4] 2> 874409 INFO (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario RunSolrRequest path=/admin/collections&action=CREATE&name=testCollection&numShards=2&replicationFactor=2&maxShardsPerNode=5
[junit4] 2> 874416 DEBUG (ScheduledTrigger-7850-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 1
[junit4] 2> 874433 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- deactivating old nodeLost / nodeAdded markers
[junit4] 2> 874450 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
[junit4] 2> 874436 DEBUG (ScheduledTrigger-7850-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 874450 DEBUG (simCloudManagerPool-7849-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider -- simCreateCollection testCollection, currentVersion=1
[junit4] 2> 874450 DEBUG (simCloudManagerPool-7849-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=1
[junit4] 2> 874450 DEBUG (simCloudManagerPool-7849-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 1
[junit4] 2> 874460 DEBUG (simCloudManagerPool-7849-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=2
[junit4] 2> 874460 DEBUG (simCloudManagerPool-7849-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 2
[junit4] 2> 874502 DEBUG (ScheduledTrigger-7850-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 874557 DEBUG (simCloudManagerPool-7849-thread-2) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=3
[junit4] 2> 874557 DEBUG (simCloudManagerPool-7849-thread-2) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 3
[junit4] 2> 874557 DEBUG (simCloudManagerPool-7849-thread-2) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election (testCollection / shard1)
[junit4] 2> 874557 DEBUG (simCloudManagerPool-7849-thread-2) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testCollection / shard1 (currentVersion=4): {"core_node1":{}}
[junit4] 2> 874712 DEBUG (ScheduledTrigger-7850-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 874713 DEBUG (simCloudManagerPool-7849-thread-3) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=4
[junit4] 2> 874714 DEBUG (simCloudManagerPool-7849-thread-3) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 4
[junit4] 2> 874724 DEBUG (simCloudManagerPool-7849-thread-4) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=5
[junit4] 2> 874725 DEBUG (simCloudManagerPool-7849-thread-4) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 5
[junit4] 2> 874725 DEBUG (simCloudManagerPool-7849-thread-4) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election (testCollection / shard2)
[junit4] 2> 874725 DEBUG (simCloudManagerPool-7849-thread-4) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testCollection / shard2 (currentVersion=6): {"core_node3":{}}
[junit4] 2> 874725 DEBUG (simCloudManagerPool-7849-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider -- finished createCollection testCollection, currentVersion=6
[junit4] 2> 874728 INFO (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario 3. WaitCollection collection=testCollection&shards=2&replicas=2
[junit4] 2> 874728 INFO (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario WaitCollection collection=testCollection&shards=2&replicas=2
[junit4] 2> 874733 DEBUG (ScheduledTrigger-7850-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 874735 DEBUG (simCloudManagerPool-7849-thread-5) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=6
[junit4] 2> 874735 DEBUG (simCloudManagerPool-7849-thread-5) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 6
[junit4] 2> 874735 DEBUG (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=7
[junit4] 2> 874736 DEBUG (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 7
[junit4] 2> 874736 INFO (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario 4. SetShardMetrics collection=testCollection&shard=shard1&INDEX.sizeInBytes=1000000000
[junit4] 2> 874736 INFO (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario SetShardMetrics collection=testCollection&shard=shard1&INDEX.sizeInBytes=1000000000
[junit4] 2> 874737 INFO (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario 5. SetNodeMetrics nodeset=#ANY&freedisk=1.5
[junit4] 2> 874737 INFO (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario SetNodeMetrics nodeset=#ANY&freedisk=1.5
[junit4] 2> 874738 INFO (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario 6. RunSolrRequest path=/admin/collection&action=SPLITSHARD&collection=testCollection&shard=shard1&splitMethod=${method}
[junit4] 2> 874738 INFO (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario RunSolrRequest path=/admin/collection&action=SPLITSHARD&collection=testCollection&shard=shard1&splitMethod=REWRITE
[junit4] 2> 874753 DEBUG (ScheduledTrigger-7850-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 874754 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 874754 DEBUG (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
[junit4] 2> 874754 DEBUG (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now
[junit4] 2> 874754 DEBUG (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
[junit4] 2> 874754 DEBUG (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor
[junit4] 2> 874754 DEBUG (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
[junit4] 2> 874754 DEBUG (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
[junit4] 2> 874755 INFO (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario 1. CreateCluster numNodes=2
[junit4] 2> 874755 INFO (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario CreateCluster numNodes=2
[junit4] 2> 874764 DEBUG (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimClusterStateProvider --- new Overseer leader: 127.0.0.1:10004_solr
[junit4] 2> 874774 DEBUG (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=0
[junit4] 2> 874774 DEBUG (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 0
[junit4] 2> 874774 INFO (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 874774 INFO (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario 2. RunSolrRequest path=/admin/collections&action=CREATE&name=testCollection&numShards=2&replicationFactor=2&maxShardsPerNode=5
[junit4] 2> 874774 INFO (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario RunSolrRequest path=/admin/collections&action=CREATE&name=testCollection&numShards=2&replicationFactor=2&maxShardsPerNode=5
[junit4] 2> 874781 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
[junit4] 2> 874781 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
[junit4] 2> 874782 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 0
[junit4] 2> 874782 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
[junit4] 2> 874785 DEBUG (simCloudManagerPool-7855-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider -- simCreateCollection testCollection, currentVersion=1
[junit4] 2> 874785 DEBUG (simCloudManagerPool-7855-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=1
[junit4] 2> 874797 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10004_solr, 127.0.0.1:10005_solr]
[junit4] 2> 874797 DEBUG (simCloudManagerPool-7855-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 1
[junit4] 2> 874807 DEBUG (simCloudManagerPool-7855-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=2
[junit4] 2> 874808 DEBUG (simCloudManagerPool-7855-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 2
[junit4] 2> 874813 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- deactivating old nodeLost / nodeAdded markers
[junit4] 2> 874813 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
[junit4] 2> 874813 DEBUG (ScheduledTrigger-7856-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 874847 DEBUG (simCloudManagerPool-7855-thread-2) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=3
[junit4] 2> 874848 DEBUG (simCloudManagerPool-7855-thread-2) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 3
[junit4] 2> 874848 DEBUG (simCloudManagerPool-7855-thread-2) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election (testCollection / shard1)
[junit4] 2> 874848 DEBUG (simCloudManagerPool-7855-thread-2) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testCollection / shard1 (currentVersion=4): {"core_node1":{}}
[junit4] 2> 874863 DEBUG (ScheduledTrigger-7856-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 874876 DEBUG (simCloudManagerPool-7855-thread-3) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=4
[junit4] 2> 874883 DEBUG (simCloudManagerPool-7855-thread-3) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 4
[junit4] 2> 874883 DEBUG (ScheduledTrigger-7856-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 874893 DEBUG (simCloudManagerPool-7855-thread-4) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=5
[junit4] 2> 874894 DEBUG (simCloudManagerPool-7855-thread-4) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 5
[junit4] 2> 874894 DEBUG (simCloudManagerPool-7855-thread-4) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election (testCollection / shard2)
[junit4] 2> 874894 DEBUG (simCloudManagerPool-7855-thread-4) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testCollection / shard2 (currentVersion=6): {"core_node3":{}}
[junit4] 2> 874894 DEBUG (simCloudManagerPool-7855-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider -- finished createCollection testCollection, currentVersion=6
[junit4] 2> 874894 INFO (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario 3. WaitCollection collection=testCollection&shards=2&replicas=2
[junit4] 2> 874894 INFO (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario WaitCollection collection=testCollection&shards=2&replicas=2
[junit4] 2> 874904 DEBUG (ScheduledTrigger-7856-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 874905 DEBUG (simCloudManagerPool-7855-thread-5) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=6
[junit4] 2> 874906 DEBUG (simCloudManagerPool-7855-thread-5) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 6
[junit4] 2> 874906 DEBUG (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=7
[junit4] 2> 874906 DEBUG (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 7
[junit4] 2> 874906 INFO (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario 4. SetShardMetrics collection=testCollection&shard=shard1&INDEX.sizeInBytes=1000000000
[junit4] 2> 874906 INFO (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario SetShardMetrics collection=testCollection&shard=shard1&INDEX.sizeInBytes=1000000000
[junit4] 2> 874906 INFO (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario 5. SetNodeMetrics nodeset=#ANY&freedisk=1.5
[junit4] 2> 874906 INFO (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario SetNodeMetrics nodeset=#ANY&freedisk=1.5
[junit4] 2> 874906 INFO (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario 6. RunSolrRequest path=/admin/collection&action=SPLITSHARD&collection=testCollection&shard=shard1&splitMethod=${method}
[junit4] 2> 874906 INFO (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario RunSolrRequest path=/admin/collection&action=SPLITSHARD&collection=testCollection&shard=shard1&splitMethod=LINK
[junit4] 2> 874924 DEBUG (ScheduledTrigger-7856-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 874944 DEBUG (ScheduledTrigger-7856-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 874964 DEBUG (ScheduledTrigger-7856-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 874984 DEBUG (ScheduledTrigger-7856-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 875004 DEBUG (ScheduledTrigger-7856-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 875017 INFO (simCloudManagerPool-7855-thread-6) [ ] o.a.s.c.a.c.SplitShardCmd numSubShards set at: 2
[junit4] 2> 875024 DEBUG (ScheduledTrigger-7856-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 875147 DEBUG (ScheduledTrigger-7856-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 875185 DEBUG (ScheduledTrigger-7856-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 875185 DEBUG (simCloudManagerPool-7855-thread-6) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=8
[junit4] 2> 875185 DEBUG (simCloudManagerPool-7855-thread-6) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 8
[junit4] 2> 875188 DEBUG (simCloudManagerPool-7855-thread-8) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=9
[junit4] 2> 875201 DEBUG (simCloudManagerPool-7855-thread-8) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 9
[junit4] 2> 875203 DEBUG (simCloudManagerPool-7855-thread-9) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=10
[junit4] 2> 875203 DEBUG (simCloudManagerPool-7855-thread-9) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 10
[junit4] 2> 875203 DEBUG (simCloudManagerPool-7855-thread-9) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election (testCollection / shard1_1)
[junit4] 2> 875203 DEBUG (simCloudManagerPool-7855-thread-9) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testCollection / shard1_1 (currentVersion=11): {"core_node12":{}}
[junit4] 2> 875203 DEBUG (simCloudManagerPool-7855-thread-6) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=11
[junit4] 2> 875203 DEBUG (simCloudManagerPool-7855-thread-6) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 11
[junit4] 2> 875203 DEBUG (simCloudManagerPool-7855-thread-6) [ ] o.a.s.c.a.s.SimClusterStateProvider ** no leader in testCollection / shard1_0:{
[junit4] 2> "parent":"shard1",
[junit4] 2> "stateTimestamp":"1634619908610584950",
[junit4] 2> "range":"80000000-bfffffff",
[junit4] 2> "state":"construction",
[junit4] 2> "replicas":{
[junit4] 2> "core_node10":{
[junit4] 2> "core":"testCollection_shard1_0_replica_n9",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":20480,
[junit4] 2> "node_name":"127.0.0.1:10005_solr",
[junit4] 2> "base_url":"http://127.0.0.1:10005/solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":1.9073486328125E-5,
[junit4] 2> "SEARCHER.searcher.numDocs":0},
[junit4] 2> "core_node8":{
[junit4] 2> "core":"testCollection_shard1_0_replica_n7",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":20480,
[junit4] 2> "node_name":"127.0.0.1:10004_solr",
[junit4] 2> "base_url":"http://127.0.0.1:10004/solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":1.9073486328125E-5,
[junit4] 2> "SEARCHER.searcher.numDocs":0}}}
[junit4] 2> 875204 DEBUG (simCloudManagerPool-7855-thread-10) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election (testCollection / shard1_0)
[junit4] 2> 875204 DEBUG (simCloudManagerPool-7855-thread-10) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testCollection / shard1_0 (currentVersion=12): {"core_node10":{}}
[junit4] 2> 875205 DEBUG (ScheduledTrigger-7856-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 875206 DEBUG (simCloudManagerPool-7855-thread-6) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=12
[junit4] 2> 875206 DEBUG (simCloudManagerPool-7855-thread-6) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 12
[junit4] 2> 875206 INFO (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario 7. WaitCollection collection=testCollection&shards=4&withInactive=true&replicas=2&requireLeaders=true
[junit4] 2> 875206 INFO (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario WaitCollection collection=testCollection&shards=4&withInactive=true&replicas=2&requireLeaders=true
[junit4] 2> 875207 DEBUG (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=13
[junit4] 2> 875207 DEBUG (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 13
[junit4] 2> 875207 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 875208 DEBUG (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
[junit4] 2> 875208 DEBUG (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now
[junit4] 2> 875208 DEBUG (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
[junit4] 2> 875208 DEBUG (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor
[junit4] 2> 875208 DEBUG (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
[junit4] 2> 875208 DEBUG (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
[junit4] 2> 875209 INFO (TEST-TestSimScenario.testSplitShard-seed#[93D020C624824834]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testSplitShard
[junit4] 2> 875251 INFO (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testSuggestions
[junit4] 2> 875252 INFO (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario 1. CreateCluster numNodes=2
[junit4] 2> 875252 INFO (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario CreateCluster numNodes=2
[junit4] 2> 875259 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimClusterStateProvider --- new Overseer leader: 127.0.0.1:10006_solr
[junit4] 2> 875265 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
[junit4] 2> 875265 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
[junit4] 2> 875266 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 0
[junit4] 2> 875266 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
[junit4] 2> 875274 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=0
[junit4] 2> 875274 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 0
[junit4] 2> 875274 INFO (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 875274 INFO (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario 2. RunSolrRequest path=/admin/collections&action=CREATE&autoAddReplicas=true&name=testCollection&numShards=2&replicationFactor=2&maxShardsPerNode=2
[junit4] 2> 875274 INFO (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario RunSolrRequest path=/admin/collections&action=CREATE&autoAddReplicas=true&name=testCollection&numShards=2&replicationFactor=2&maxShardsPerNode=2
[junit4] 2> 875285 DEBUG (simCloudManagerPool-7861-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider -- simCreateCollection testCollection, currentVersion=1
[junit4] 2> 875285 DEBUG (simCloudManagerPool-7861-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=1
[junit4] 2> 875288 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10006_solr, 127.0.0.1:10007_solr]
[junit4] 2> 875289 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- deactivating old nodeLost / nodeAdded markers
[junit4] 2> 875289 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
[junit4] 2> 875309 DEBUG (simCloudManagerPool-7861-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 1
[junit4] 2> 875309 DEBUG (ScheduledTrigger-7862-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 875320 DEBUG (simCloudManagerPool-7861-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=2
[junit4] 2> 875320 DEBUG (simCloudManagerPool-7861-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 2
[junit4] 2> 875334 DEBUG (simCloudManagerPool-7861-thread-2) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=3
[junit4] 2> 875334 DEBUG (simCloudManagerPool-7861-thread-2) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 3
[junit4] 2> 875334 DEBUG (simCloudManagerPool-7861-thread-2) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election (testCollection / shard1)
[junit4] 2> 875334 DEBUG (simCloudManagerPool-7861-thread-2) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testCollection / shard1 (currentVersion=4): {"core_node1":{}}
[junit4] 2> 875393 DEBUG (ScheduledTrigger-7862-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 875397 DEBUG (simCloudManagerPool-7861-thread-4) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=4
[junit4] 2> 875397 DEBUG (simCloudManagerPool-7861-thread-4) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 4
[junit4] 2> 875397 DEBUG (simCloudManagerPool-7861-thread-4) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election (testCollection / shard2)
[junit4] 2> 875397 DEBUG (simCloudManagerPool-7861-thread-4) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testCollection / shard2 (currentVersion=5): {"core_node3":{}}
[junit4] 2> 875408 DEBUG (simCloudManagerPool-7861-thread-3) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=5
[junit4] 2> 875408 DEBUG (simCloudManagerPool-7861-thread-3) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 5
[junit4] 2> 875408 DEBUG (simCloudManagerPool-7861-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider -- finished createCollection testCollection, currentVersion=6
[junit4] 2> 875408 INFO (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario 3. WaitCollection collection=testCollection&shards=2&replicas=2
[junit4] 2> 875408 INFO (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario WaitCollection collection=testCollection&shards=2&replicas=2
[junit4] 2> 875413 DEBUG (ScheduledTrigger-7862-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 875418 DEBUG (simCloudManagerPool-7861-thread-5) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=6
[junit4] 2> 875419 DEBUG (simCloudManagerPool-7861-thread-5) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 6
[junit4] 2> 875419 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=7
[junit4] 2> 875419 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 7
[junit4] 2> 875420 INFO (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario 4. CtxSet key=myNode&value=${_random_node_}
[junit4] 2> 875420 INFO (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario CtxSet key=myNode&value=127.0.0.1:10006_solr
[junit4] 2> 875420 INFO (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario 5. RunSolrRequest path=/admin/collections&action=ADDREPLICA&collection=testCollection&shard=shard1&node=${myNode}
[junit4] 2> 875420 INFO (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario RunSolrRequest path=/admin/collections&action=ADDREPLICA&collection=testCollection&shard=shard1&node=127.0.0.1:10006_solr
[junit4] 2> 875421 INFO (simCloudManagerPool-7861-thread-6) [ ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:10006_solr for creating new replica of shard shard1 for collection testCollection
[junit4] 2> 875421 INFO (simCloudManagerPool-7861-thread-6) [ ] o.a.s.c.a.c.AddReplicaCmd Returning CreateReplica command.
[junit4] 2> 875432 DEBUG (simCloudManagerPool-7861-thread-6) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=8
[junit4] 2> 875432 DEBUG (simCloudManagerPool-7861-thread-6) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 8
[junit4] 2> 875432 INFO (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario 6. RunSolrRequest path=/admin/collections&action=ADDREPLICA&collection=testCollection&shard=shard1&node=${myNode}
[junit4] 2> 875432 INFO (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario RunSolrRequest path=/admin/collections&action=ADDREPLICA&collection=testCollection&shard=shard1&node=127.0.0.1:10006_solr
[junit4] 2> 875432 DEBUG (simCloudManagerPool-7861-thread-7) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=9
[junit4] 2> 875433 DEBUG (simCloudManagerPool-7861-thread-7) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 9
[junit4] 2> 875433 INFO (simCloudManagerPool-7861-thread-7) [ ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:10006_solr for creating new replica of shard shard1 for collection testCollection
[junit4] 2> 875433 INFO (simCloudManagerPool-7861-thread-7) [ ] o.a.s.c.a.c.AddReplicaCmd Returning CreateReplica command.
[junit4] 2> 875434 DEBUG (ScheduledTrigger-7862-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 875443 DEBUG (simCloudManagerPool-7861-thread-7) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=10
[junit4] 2> 875443 DEBUG (simCloudManagerPool-7861-thread-7) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 10
[junit4] 2> 875444 INFO (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario 7. LoopOp iterations=${iterative}
[junit4] 2> 875444 INFO (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario LoopOp iterations=0
[junit4] 2> 875444 INFO (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario 8. LoopOp iterations=${justCalc}
[junit4] 2> 875444 INFO (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario LoopOp iterations=1
[junit4] 2> 875444 INFO (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario * iter 1 :
[junit4] 2> 875444 INFO (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario - CalculateSuggestions )
[junit4] 2> 875450 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=11
[junit4] 2> 875450 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 11
[junit4] 2> 875454 DEBUG (ScheduledTrigger-7862-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 875460 INFO (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario - 1 suggestions
[junit4] 2> 875460 INFO (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario - SaveSnapshot path=/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-Tests-8.10/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.sim.TestSimScenario_93D020C624824834-001/tempDir-002/snapshot/0)
[junit4] 2> 875474 DEBUG (ScheduledTrigger-7862-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 875591 DEBUG (ScheduledTrigger-7862-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 875611 DEBUG (ScheduledTrigger-7862-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 875620 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SnapshotDistribStateManager - copying 22 resources from SimDistribStateManager
[junit4] 2> 875631 DEBUG (ScheduledTrigger-7862-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 875651 DEBUG (ScheduledTrigger-7862-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 875652 INFO (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario 9. Dump redact=true
[junit4] 2> 875652 INFO (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario Dump redact=true
[junit4] 2> 875653 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SnapshotDistribStateManager - copying 22 resources from SimDistribStateManager
[junit4] 2> 875656 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
[junit4] 2> 875656 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now
[junit4] 2> 875656 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 875657 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
[junit4] 2> 875657 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor
[junit4] 2> 875657 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
[junit4] 2> 875657 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
[junit4] 2> 875658 INFO (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario 1. CreateCluster numNodes=2
[junit4] 2> 875658 INFO (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario CreateCluster numNodes=2
[junit4] 2> 875664 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimClusterStateProvider --- new Overseer leader: 127.0.0.1:10008_solr
[junit4] 2> 875665 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
[junit4] 2> 875665 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
[junit4] 2> 875665 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 0
[junit4] 2> 875665 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
[junit4] 2> 875665 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=0
[junit4] 2> 875665 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 0
[junit4] 2> 875665 INFO (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 875666 INFO (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario 2. RunSolrRequest path=/admin/collections&action=CREATE&autoAddReplicas=true&name=testCollection&numShards=2&replicationFactor=2&maxShardsPerNode=2
[junit4] 2> 875666 INFO (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario RunSolrRequest path=/admin/collections&action=CREATE&autoAddReplicas=true&name=testCollection&numShards=2&replicationFactor=2&maxShardsPerNode=2
[junit4] 2> 875666 DEBUG (simCloudManagerPool-7867-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider -- simCreateCollection testCollection, currentVersion=1
[junit4] 2> 875666 DEBUG (simCloudManagerPool-7867-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=1
[junit4] 2> 875666 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10009_solr, 127.0.0.1:10008_solr]
[junit4] 2> 875666 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- deactivating old nodeLost / nodeAdded markers
[junit4] 2> 875666 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
[junit4] 2> 875669 DEBUG (simCloudManagerPool-7867-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 1
[junit4] 2> 875669 DEBUG (ScheduledTrigger-7868-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 875679 DEBUG (simCloudManagerPool-7867-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=2
[junit4] 2> 875679 DEBUG (simCloudManagerPool-7867-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 2
[junit4] 2> 875690 DEBUG (ScheduledTrigger-7868-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 875695 DEBUG (simCloudManagerPool-7867-thread-2) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=3
[junit4] 2> 875695 DEBUG (simCloudManagerPool-7867-thread-2) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 3
[junit4] 2> 875695 DEBUG (simCloudManagerPool-7867-thread-2) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election (testCollection / shard1)
[junit4] 2> 875695 DEBUG (simCloudManagerPool-7867-thread-2) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testCollection / shard1 (currentVersion=4): {"core_node1":{}}
[junit4] 2> 875705 DEBUG (simCloudManagerPool-7867-thread-3) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=4
[junit4] 2> 875706 DEBUG (simCloudManagerPool-7867-thread-3) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 4
[junit4] 2> 875710 DEBUG (ScheduledTrigger-7868-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 875716 DEBUG (simCloudManagerPool-7867-thread-4) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=5
[junit4] 2> 875717 DEBUG (simCloudManagerPool-7867-thread-4) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 5
[junit4] 2> 875717 DEBUG (simCloudManagerPool-7867-thread-4) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election (testCollection / shard2)
[junit4] 2> 875717 DEBUG (simCloudManagerPool-7867-thread-4) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testCollection / shard2 (currentVersion=6): {"core_node3":{}}
[junit4] 2> 875717 DEBUG (simCloudManagerPool-7867-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider -- finished createCollection testCollection, currentVersion=6
[junit4] 2> 875717 INFO (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario 3. WaitCollection collection=testCollection&shards=2&replicas=2
[junit4] 2> 875717 INFO (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario WaitCollection collection=testCollection&shards=2&replicas=2
[junit4] 2> 875727 DEBUG (simCloudManagerPool-7867-thread-5) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=6
[junit4] 2> 875728 DEBUG (simCloudManagerPool-7867-thread-5) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 6
[junit4] 2> 875728 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=7
[junit4] 2> 875728 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 7
[junit4] 2> 875728 INFO (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario 4. CtxSet key=myNode&value=${_random_node_}
[junit4] 2> 875728 INFO (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario CtxSet key=myNode&value=127.0.0.1:10009_solr
[junit4] 2> 875728 INFO (TEST-TestSimScenario.testSuggestions-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario 5. RunSolrRequest path=/admin/collections&action=ADDREPLICA&collection=testCollection&shard=shard1&node=${myNode}
[junit4] 2> 875728 INFO (TEST-TestSimScenario.testSuggestions-seed#[93D020C
[...truncated too long message...]
: 100
[junit4] 2> 879292 DEBUG (ScheduledTrigger-7874-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100 and last live nodes: 100
[junit4] 2> 879312 DEBUG (ScheduledTrigger-7874-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100 and last live nodes: 100
[junit4] 2> 879332 DEBUG (ScheduledTrigger-7874-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100 and last live nodes: 100
[junit4] 2> 879352 DEBUG (ScheduledTrigger-7874-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100 and last live nodes: 100
[junit4] 2> 879373 DEBUG (ScheduledTrigger-7874-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100 and last live nodes: 100
[junit4] 2> 879393 DEBUG (ScheduledTrigger-7874-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100 and last live nodes: 100
[junit4] 2> 879413 DEBUG (ScheduledTrigger-7874-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100 and last live nodes: 100
[junit4] 2> 879433 DEBUG (ScheduledTrigger-7874-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100 and last live nodes: 100
[junit4] 2> 879454 DEBUG (ScheduledTrigger-7874-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100 and last live nodes: 100
[junit4] 2> 879474 DEBUG (ScheduledTrigger-7874-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100 and last live nodes: 100
[junit4] 2> 879481 INFO (TEST-TestSimScenario.testIndexing-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario 8. WaitEvent trigger=indexSizeTrigger&wait=60
[junit4] 2> 879481 INFO (TEST-TestSimScenario.testIndexing-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario WaitEvent trigger=indexSizeTrigger&wait=60
[junit4] 2> 879481 INFO (TEST-TestSimScenario.testIndexing-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario 9. Assert condition=not_null&key=_trigger_event_indexSizeTrigger
[junit4] 2> 879481 INFO (TEST-TestSimScenario.testIndexing-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario Assert condition=not_null&key=_trigger_event_indexSizeTrigger
[junit4] 2> 879495 DEBUG (ScheduledTrigger-7874-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100 and last live nodes: 100
[junit4] 2> 879497 INFO (TEST-TestSimScenario.testIndexing-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario 10. Assert condition=equals&key=_trigger_event_indexSizeTrigger/eventType&expected=INDEXSIZE
[junit4] 2> 879498 INFO (TEST-TestSimScenario.testIndexing-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario Assert condition=equals&key=_trigger_event_indexSizeTrigger/eventType&expected=INDEXSIZE
[junit4] 2> 879498 INFO (TEST-TestSimScenario.testIndexing-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario 11. Assert condition=equals&key=_trigger_event_indexSizeTrigger/properties/requestedOps[0]/action&expected=SPLITSHARD
[junit4] 2> 879498 INFO (TEST-TestSimScenario.testIndexing-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario Assert condition=equals&key=_trigger_event_indexSizeTrigger/properties/requestedOps[0]/action&expected=SPLITSHARD
[junit4] 2> 879498 INFO (TEST-TestSimScenario.testIndexing-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario 12. WaitCollection collection=testCollection&shards=6&withInactive=true&requireLeaders=false&replicas=2
[junit4] 2> 879498 INFO (TEST-TestSimScenario.testIndexing-seed#[93D020C624824834]) [ ] o.a.s.c.a.s.SimScenario WaitCollection collection=testCollection&shards=6&withInactive=true&requireLeaders=false&replicas=2
[junit4] 2> 879499 DEBUG (TEST-TestSimScenario.testIndexing-seed#[93D020C624824834]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
[junit4] 2> 879499 DEBUG (TEST-TestSimScenario.testIndexing-seed#[93D020C624824834]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now
[junit4] 2> 879499 DEBUG (TEST-TestSimScenario.testIndexing-seed#[93D020C624824834]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
[junit4] 2> 879499 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 879501 DEBUG (TEST-TestSimScenario.testIndexing-seed#[93D020C624824834]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor
[junit4] 2> 879499 WARN (ScheduledTrigger-7874-thread-3) [ ] o.a.s.c.a.IndexSizeTrigger Error running trigger indexSizeTrigger
[junit4] 2> => java.io.IOException: already closed
[junit4] 2> at org.apache.solr.cloud.autoscaling.sim.SimClusterStateProvider.ensureNotClosed(SimClusterStateProvider.java:2503)
[junit4] 2> java.io.IOException: already closed
[junit4] 2> at org.apache.solr.cloud.autoscaling.sim.SimClusterStateProvider.ensureNotClosed(SimClusterStateProvider.java:2503) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.sim.SimClusterStateProvider.getClusterState(SimClusterStateProvider.java:2394) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.IndexSizeTrigger.run(IndexSizeTrigger.java:269) [java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers$TriggerWrapper.run(ScheduledTriggers.java:660) [java/:?]
[junit4] 2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [?:1.8.0_291]
[junit4] 2> at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) [?:1.8.0_291]
[junit4] 2> at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) [?:1.8.0_291]
[junit4] 2> at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) [?:1.8.0_291]
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_291]
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_291]
[junit4] 2> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_291]
[junit4] 2> 879501 DEBUG (TEST-TestSimScenario.testIndexing-seed#[93D020C624824834]) [ ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
[junit4] 2> 879501 DEBUG (TEST-TestSimScenario.testIndexing-seed#[93D020C624824834]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
[junit4] 2> 879502 INFO (TEST-TestSimScenario.testIndexing-seed#[93D020C624824834]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testIndexing
[junit4] 2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-Tests-8.10/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.sim.TestSimScenario_93D020C624824834-001
[junit4] 2> NOTE: test params are: codec=CheapBastard, sim=Asserting(RandomSimilarity(queryNorm=true): {}), locale=et, timezone=America/Indiana/Vincennes
[junit4] 2> NOTE: Linux 5.4.0-45-generic amd64/Oracle Corporation 1.8.0_291 (64-bit)/cpus=4,threads=1,free=110936200,total=504365056
[junit4] 2> NOTE: All tests run in this JVM: [CurrencyRangeFacetCloudTest, HDFSCollectionsAPITest, TimeZoneUtilsTest, TestCoreAdminApis, DocValuesMissingTest, SpellPossibilityIteratorTest, LeaderElectionContextKeyTest, TestTlogReplica, PeerSyncWithLeaderAndIndexFingerprintCachingTest, PKIAuthenticationIntegrationTest, TestFilteredDocIdSet, TestCodecSupport, LoggingHandlerTest, TimeRoutedAliasUpdateProcessorTest, TestConfigSetImmutable, TestNumericRangeQuery32, CheckHdfsIndexTest, TestCloudPivotFacet, NotRequiredUniqueKeyTest, RecoveryZkTest, MoreLikeThisComponentTest, TestManagedResource, TestStressCloudBlindAtomicUpdates, RandomizedTaggerTest, TestRandomFlRTGCloud, TestComplexPhraseQParserPlugin, TestExplainDocTransformer, ScheduledTriggerTest, TestJsonRequest, PrimUtilsTest, TestObjectReleaseTracker, MetricUtilsTest, TestHalfAndHalfDocValues, TestSolrCachePerf, ExecutePlanActionTest, MoveReplicaHDFSFailoverTest, SolrCmdDistributorTest, TestLegacyFieldReuse, RAMDirectoryFactoryTest, TestBooleanSimilarityFactory, TestDynamicLoadingUrl, CopyFieldTest, SpatialHeatmapFacetsTest, BigEndianAscendingWordSerializerTest, TestXIncludeConfig, SuggesterFSTTest, ChaosMonkeySafeLeaderWithPullReplicasTest, XmlInterpolationTest, HdfsRestartWhileUpdatingTest, TriLevelCompositeIdRoutingTest, TestReload, TestCloudManagedSchema, DirectSolrConnectionTest, ShardRoutingCustomTest, TestCloudConsistency, ExternalFileFieldSortTest, TestCircuitBreaker, ZkCollectionPropsCachingTest, TestPolicyCloud, BlockJoinFacetSimpleTest, AnalysisAfterCoreReloadTest, CdcrReplicationHandlerTest, TestFieldResource, DaemonStreamApiTest, TestCloudSearcherWarming, TermVectorComponentDistributedTest, TestRuleBasedAuthorizationWithKerberos, TriggerSetPropertiesIntegrationTest, TestExpandComponent, CircularListTest, TestFreeTextSuggestions, TestSurroundQueryParser, SpellCheckComponentTest, HdfsCollectionsAPIDistributedZkTest, SimpleCollectionCreateDeleteTest, TestConfigSetsAPIShareSchema, RuleEngineTest, IgnoreLargeDocumentProcessorFactoryTest, TestSimScenario]
[junit4] Completed [394/947 (1!)] on J0 in 7.01s, 4 tests, 1 error <<< FAILURES!
[...truncated 56071 lines...]