You are viewing a plain text version of this content. The canonical link for it is here.
Posted to builds@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2021/03/25 18:41:52 UTC
[JENKINS] Lucene » Lucene-Solr-Tests-8.x - Build # 1872 - Still Unstable!
Build: https://ci-builds.apache.org/job/Lucene/job/Lucene-Solr-Tests-8.x/1872/
1 tests failed.
FAILED: org.apache.solr.cloud.autoscaling.sim.TestSimScenario.testAutoAddReplicas
Error Message:
Timed out waiting for trigger .auto_add_replicas to fire after simulated 60s (real 1200ms).
Stack Trace:
java.io.IOException: Timed out waiting for trigger .auto_add_replicas to fire after simulated 60s (real 1200ms).
at __randomizedtesting.SeedInfo.seed([6AE4FA6F8CF310EA:E3440913F8F8DCC3]:0)
at org.apache.solr.cloud.autoscaling.sim.SimScenario$SimWaitListener.wait(SimScenario.java:705)
at org.apache.solr.cloud.autoscaling.sim.SimScenario$WaitEvent.execute(SimScenario.java:771)
at org.apache.solr.cloud.autoscaling.sim.SimScenario.run(SimScenario.java:1119)
at org.apache.solr.cloud.autoscaling.sim.TestSimScenario.testAutoAddReplicas(TestSimScenario.java:57)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 13968 lines...]
[junit4] Suite: org.apache.solr.cloud.autoscaling.sim.TestSimScenario
[junit4] 2> 621543 INFO (SUITE-TestSimScenario-seed#[6AE4FA6F8CF310EA]-worker) [ ] o.a.s.SolrTestCase Setting 'solr.default.confdir' system property to test-framework derived value of '/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-Tests-8.x/solr/server/solr/configsets/_default/conf'
[junit4] 2> 621543 INFO (SUITE-TestSimScenario-seed#[6AE4FA6F8CF310EA]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 621545 INFO (SUITE-TestSimScenario-seed#[6AE4FA6F8CF310EA]-worker) [ ] o.a.s.SolrTestCaseJ4 Created dataDir: /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J3/temp/solr.cloud.autoscaling.sim.TestSimScenario_6AE4FA6F8CF310EA-001/data-dir-66-001
[junit4] 2> 621545 WARN (SUITE-TestSimScenario-seed#[6AE4FA6F8CF310EA]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=6 numCloses=6
[junit4] 2> 621545 INFO (SUITE-TestSimScenario-seed#[6AE4FA6F8CF310EA]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 621549 INFO (SUITE-TestSimScenario-seed#[6AE4FA6F8CF310EA]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 621552 INFO (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testSplitShard
[junit4] 2> 621604 INFO (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario 1. CreateCluster numNodes=2
[junit4] 2> 621604 INFO (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario CreateCluster numNodes=2
[junit4] 2> 621643 INFO (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.SolrPaths solr home defaulted to 'solr/' (could not find system property or JNDI)
[junit4] 2> 621928 DEBUG (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimClusterStateProvider --- new Overseer leader: 127.0.0.1:10000_solr
[junit4] 2> 621928 DEBUG (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=0
[junit4] 2> 621929 DEBUG (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 0
[junit4] 2> 621929 INFO (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 621929 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
[junit4] 2> 621930 INFO (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario 2. RunSolrRequest path=/admin/collections&action=CREATE&name=testCollection&numShards=2&replicationFactor=2&maxShardsPerNode=5
[junit4] 2> 621930 INFO (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario RunSolrRequest path=/admin/collections&action=CREATE&name=testCollection&numShards=2&replicationFactor=2&maxShardsPerNode=5
[junit4] 2> 621931 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
[junit4] 2> 621931 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 0
[junit4] 2> 621931 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
[junit4] 2> 621934 DEBUG (simCloudManagerPool-3665-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider -- simCreateCollection testCollection, currentVersion=1
[junit4] 2> 621934 DEBUG (simCloudManagerPool-3665-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=1
[junit4] 2> 621934 DEBUG (simCloudManagerPool-3665-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 1
[junit4] 2> 621946 DEBUG (simCloudManagerPool-3665-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=2
[junit4] 2> 621949 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10000_solr]
[junit4] 2> 621957 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- deactivating old nodeLost / nodeAdded markers
[junit4] 2> 621957 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
[junit4] 2> 621957 DEBUG (ScheduledTrigger-3666-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 621959 DEBUG (simCloudManagerPool-3665-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 2
[junit4] 2> 621978 DEBUG (ScheduledTrigger-3666-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 621992 DEBUG (simCloudManagerPool-3665-thread-2) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=3
[junit4] 2> 621992 DEBUG (simCloudManagerPool-3665-thread-2) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 3
[junit4] 2> 621992 DEBUG (simCloudManagerPool-3665-thread-2) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election (testCollection / shard1)
[junit4] 2> 621993 DEBUG (simCloudManagerPool-3665-thread-2) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testCollection / shard1 (currentVersion=4): {"core_node1":{}}
[junit4] 2> 621999 DEBUG (ScheduledTrigger-3666-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 622007 DEBUG (simCloudManagerPool-3665-thread-3) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=4
[junit4] 2> 622008 DEBUG (simCloudManagerPool-3665-thread-3) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 4
[junit4] 2> 622018 DEBUG (simCloudManagerPool-3665-thread-4) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=5
[junit4] 2> 622019 DEBUG (simCloudManagerPool-3665-thread-4) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 5
[junit4] 2> 622019 DEBUG (simCloudManagerPool-3665-thread-4) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election (testCollection / shard2)
[junit4] 2> 622019 DEBUG (simCloudManagerPool-3665-thread-4) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testCollection / shard2 (currentVersion=6): {"core_node3":{}}
[junit4] 2> 622019 DEBUG (simCloudManagerPool-3665-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider -- finished createCollection testCollection, currentVersion=6
[junit4] 2> 622019 DEBUG (ScheduledTrigger-3666-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 622019 INFO (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario 3. WaitCollection collection=testCollection&shards=2&replicas=2
[junit4] 2> 622019 INFO (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario WaitCollection collection=testCollection&shards=2&replicas=2
[junit4] 2> 622029 DEBUG (simCloudManagerPool-3665-thread-5) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=6
[junit4] 2> 622029 DEBUG (simCloudManagerPool-3665-thread-5) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 6
[junit4] 2> 622029 DEBUG (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=7
[junit4] 2> 622030 DEBUG (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 7
[junit4] 2> 622030 INFO (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario 4. SetShardMetrics collection=testCollection&shard=shard1&INDEX.sizeInBytes=1000000000
[junit4] 2> 622030 INFO (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario SetShardMetrics collection=testCollection&shard=shard1&INDEX.sizeInBytes=1000000000
[junit4] 2> 622031 INFO (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario 5. SetNodeMetrics nodeset=#ANY&freedisk=1.5
[junit4] 2> 622031 INFO (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario SetNodeMetrics nodeset=#ANY&freedisk=1.5
[junit4] 2> 622032 INFO (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario 6. RunSolrRequest path=/admin/collection&action=SPLITSHARD&collection=testCollection&shard=shard1&splitMethod=${method}
[junit4] 2> 622032 INFO (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario RunSolrRequest path=/admin/collection&action=SPLITSHARD&collection=testCollection&shard=shard1&splitMethod=REWRITE
[junit4] 2> 622038 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 622039 DEBUG (ScheduledTrigger-3666-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 622039 DEBUG (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
[junit4] 2> 622039 DEBUG (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now
[junit4] 2> 622040 DEBUG (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
[junit4] 2> 622040 DEBUG (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor
[junit4] 2> 622040 DEBUG (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
[junit4] 2> 622040 DEBUG (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
[junit4] 2> 622040 INFO (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario 1. CreateCluster numNodes=2
[junit4] 2> 622040 INFO (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario CreateCluster numNodes=2
[junit4] 2> 622044 DEBUG (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimClusterStateProvider --- new Overseer leader: 127.0.0.1:10002_solr
[junit4] 2> 622044 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
[junit4] 2> 622044 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
[junit4] 2> 622045 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 0
[junit4] 2> 622045 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
[junit4] 2> 622045 DEBUG (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=0
[junit4] 2> 622045 DEBUG (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 0
[junit4] 2> 622045 INFO (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 622045 INFO (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario 2. RunSolrRequest path=/admin/collections&action=CREATE&name=testCollection&numShards=2&replicationFactor=2&maxShardsPerNode=5
[junit4] 2> 622045 INFO (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario RunSolrRequest path=/admin/collections&action=CREATE&name=testCollection&numShards=2&replicationFactor=2&maxShardsPerNode=5
[junit4] 2> 622051 DEBUG (simCloudManagerPool-3671-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider -- simCreateCollection testCollection, currentVersion=1
[junit4] 2> 622051 DEBUG (simCloudManagerPool-3671-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=1
[junit4] 2> 622059 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10002_solr, 127.0.0.1:10003_solr]
[junit4] 2> 622059 DEBUG (simCloudManagerPool-3671-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 1
[junit4] 2> 622062 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- deactivating old nodeLost / nodeAdded markers
[junit4] 2> 622062 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
[junit4] 2> 622062 DEBUG (ScheduledTrigger-3672-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 622072 DEBUG (simCloudManagerPool-3671-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=2
[junit4] 2> 622072 DEBUG (simCloudManagerPool-3671-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 2
[junit4] 2> 622090 DEBUG (ScheduledTrigger-3672-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 622100 DEBUG (simCloudManagerPool-3671-thread-2) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=3
[junit4] 2> 622100 DEBUG (simCloudManagerPool-3671-thread-2) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 3
[junit4] 2> 622100 DEBUG (simCloudManagerPool-3671-thread-2) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election (testCollection / shard1)
[junit4] 2> 622100 DEBUG (simCloudManagerPool-3671-thread-2) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testCollection / shard1 (currentVersion=4): {"core_node1":{}}
[junit4] 2> 622111 DEBUG (ScheduledTrigger-3672-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 622111 DEBUG (simCloudManagerPool-3671-thread-3) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=4
[junit4] 2> 622112 DEBUG (simCloudManagerPool-3671-thread-3) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 4
[junit4] 2> 622122 DEBUG (simCloudManagerPool-3671-thread-4) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=5
[junit4] 2> 622123 DEBUG (simCloudManagerPool-3671-thread-4) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 5
[junit4] 2> 622123 DEBUG (simCloudManagerPool-3671-thread-4) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election (testCollection / shard2)
[junit4] 2> 622123 DEBUG (simCloudManagerPool-3671-thread-4) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testCollection / shard2 (currentVersion=6): {"core_node3":{}}
[junit4] 2> 622123 DEBUG (simCloudManagerPool-3671-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider -- finished createCollection testCollection, currentVersion=6
[junit4] 2> 622123 INFO (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario 3. WaitCollection collection=testCollection&shards=2&replicas=2
[junit4] 2> 622123 INFO (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario WaitCollection collection=testCollection&shards=2&replicas=2
[junit4] 2> 622131 DEBUG (ScheduledTrigger-3672-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 622134 DEBUG (simCloudManagerPool-3671-thread-5) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=6
[junit4] 2> 622137 DEBUG (simCloudManagerPool-3671-thread-5) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 6
[junit4] 2> 622137 DEBUG (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=7
[junit4] 2> 622138 DEBUG (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 7
[junit4] 2> 622138 INFO (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario 4. SetShardMetrics collection=testCollection&shard=shard1&INDEX.sizeInBytes=1000000000
[junit4] 2> 622138 INFO (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario SetShardMetrics collection=testCollection&shard=shard1&INDEX.sizeInBytes=1000000000
[junit4] 2> 622138 INFO (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario 5. SetNodeMetrics nodeset=#ANY&freedisk=1.5
[junit4] 2> 622138 INFO (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario SetNodeMetrics nodeset=#ANY&freedisk=1.5
[junit4] 2> 622138 INFO (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario 6. RunSolrRequest path=/admin/collection&action=SPLITSHARD&collection=testCollection&shard=shard1&splitMethod=${method}
[junit4] 2> 622138 INFO (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario RunSolrRequest path=/admin/collection&action=SPLITSHARD&collection=testCollection&shard=shard1&splitMethod=LINK
[junit4] 2> 622151 DEBUG (ScheduledTrigger-3672-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 622171 DEBUG (ScheduledTrigger-3672-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 622191 DEBUG (ScheduledTrigger-3672-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 622211 DEBUG (ScheduledTrigger-3672-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 622232 DEBUG (ScheduledTrigger-3672-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 622243 INFO (simCloudManagerPool-3671-thread-6) [ ] o.a.s.c.a.c.SplitShardCmd numSubShards set at: 2
[junit4] 2> 622252 DEBUG (ScheduledTrigger-3672-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 622272 DEBUG (ScheduledTrigger-3672-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 622285 DEBUG (simCloudManagerPool-3671-thread-6) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=8
[junit4] 2> 622285 DEBUG (simCloudManagerPool-3671-thread-6) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 8
[junit4] 2> 622292 DEBUG (ScheduledTrigger-3672-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 622508 DEBUG (ScheduledTrigger-3672-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 622524 DEBUG (simCloudManagerPool-3671-thread-8) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=9
[junit4] 2> 622540 DEBUG (simCloudManagerPool-3671-thread-8) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 9
[junit4] 2> 622541 DEBUG (simCloudManagerPool-3671-thread-9) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=10
[junit4] 2> 622541 DEBUG (ScheduledTrigger-3672-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 622541 DEBUG (simCloudManagerPool-3671-thread-9) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 10
[junit4] 2> 622541 DEBUG (simCloudManagerPool-3671-thread-9) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election (testCollection / shard1_1)
[junit4] 2> 622541 DEBUG (simCloudManagerPool-3671-thread-9) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testCollection / shard1_1 (currentVersion=11): {"core_node12":{}}
[junit4] 2> 622542 DEBUG (simCloudManagerPool-3671-thread-6) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=11
[junit4] 2> 622542 DEBUG (simCloudManagerPool-3671-thread-6) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 11
[junit4] 2> 622542 DEBUG (simCloudManagerPool-3671-thread-6) [ ] o.a.s.c.a.s.SimClusterStateProvider ** no leader in testCollection / shard1_0:{
[junit4] 2> "parent":"shard1",
[junit4] 2> "stateTimestamp":"1616694765041868950",
[junit4] 2> "range":"80000000-bfffffff",
[junit4] 2> "state":"construction",
[junit4] 2> "replicas":{
[junit4] 2> "core_node10":{
[junit4] 2> "core":"testCollection_shard1_0_replica_n9",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":20480,
[junit4] 2> "node_name":"127.0.0.1:10002_solr",
[junit4] 2> "base_url":"http://127.0.0.1:10002/solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":1.9073486328125E-5,
[junit4] 2> "SEARCHER.searcher.numDocs":0},
[junit4] 2> "core_node8":{
[junit4] 2> "core":"testCollection_shard1_0_replica_n7",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":20480,
[junit4] 2> "node_name":"127.0.0.1:10003_solr",
[junit4] 2> "base_url":"http://127.0.0.1:10003/solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":1.9073486328125E-5,
[junit4] 2> "SEARCHER.searcher.numDocs":0}}}
[junit4] 2> 622544 DEBUG (simCloudManagerPool-3671-thread-6) [ ] o.a.s.c.a.s.SimClusterStateProvider ** no leader in testCollection / shard1_0:{
[junit4] 2> "parent":"shard1",
[junit4] 2> "stateTimestamp":"1616694765041868950",
[junit4] 2> "range":"80000000-bfffffff",
[junit4] 2> "state":"construction",
[junit4] 2> "replicas":{
[junit4] 2> "core_node10":{
[junit4] 2> "core":"testCollection_shard1_0_replica_n9",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":20480,
[junit4] 2> "node_name":"127.0.0.1:10002_solr",
[junit4] 2> "base_url":"http://127.0.0.1:10002/solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":1.9073486328125E-5,
[junit4] 2> "SEARCHER.searcher.numDocs":0},
[junit4] 2> "core_node8":{
[junit4] 2> "core":"testCollection_shard1_0_replica_n7",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":20480,
[junit4] 2> "node_name":"127.0.0.1:10003_solr",
[junit4] 2> "base_url":"http://127.0.0.1:10003/solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":1.9073486328125E-5,
[junit4] 2> "SEARCHER.searcher.numDocs":0}}}
[junit4] 2> 622548 DEBUG (simCloudManagerPool-3671-thread-6) [ ] o.a.s.c.a.s.SimClusterStateProvider ** no leader in testCollection / shard1_0:{
[junit4] 2> "parent":"shard1",
[junit4] 2> "stateTimestamp":"1616694765041868950",
[junit4] 2> "range":"80000000-bfffffff",
[junit4] 2> "state":"construction",
[junit4] 2> "replicas":{
[junit4] 2> "core_node10":{
[junit4] 2> "core":"testCollection_shard1_0_replica_n9",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":20480,
[junit4] 2> "node_name":"127.0.0.1:10002_solr",
[junit4] 2> "base_url":"http://127.0.0.1:10002/solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":1.9073486328125E-5,
[junit4] 2> "SEARCHER.searcher.numDocs":0},
[junit4] 2> "core_node8":{
[junit4] 2> "core":"testCollection_shard1_0_replica_n7",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":20480,
[junit4] 2> "node_name":"127.0.0.1:10003_solr",
[junit4] 2> "base_url":"http://127.0.0.1:10003/solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":1.9073486328125E-5,
[junit4] 2> "SEARCHER.searcher.numDocs":0}}}
[junit4] 2> 622550 DEBUG (simCloudManagerPool-3671-thread-6) [ ] o.a.s.c.a.s.SimClusterStateProvider ** no leader in testCollection / shard1_0:{
[junit4] 2> "parent":"shard1",
[junit4] 2> "stateTimestamp":"1616694765041868950",
[junit4] 2> "range":"80000000-bfffffff",
[junit4] 2> "state":"construction",
[junit4] 2> "replicas":{
[junit4] 2> "core_node10":{
[junit4] 2> "core":"testCollection_shard1_0_replica_n9",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":20480,
[junit4] 2> "node_name":"127.0.0.1:10002_solr",
[junit4] 2> "base_url":"http://127.0.0.1:10002/solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":1.9073486328125E-5,
[junit4] 2> "SEARCHER.searcher.numDocs":0},
[junit4] 2> "core_node8":{
[junit4] 2> "core":"testCollection_shard1_0_replica_n7",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":20480,
[junit4] 2> "node_name":"127.0.0.1:10003_solr",
[junit4] 2> "base_url":"http://127.0.0.1:10003/solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":1.9073486328125E-5,
[junit4] 2> "SEARCHER.searcher.numDocs":0}}}
[junit4] 2> 622552 DEBUG (simCloudManagerPool-3671-thread-6) [ ] o.a.s.c.a.s.SimClusterStateProvider ** no leader in testCollection / shard1_0:{
[junit4] 2> "parent":"shard1",
[junit4] 2> "stateTimestamp":"1616694765041868950",
[junit4] 2> "range":"80000000-bfffffff",
[junit4] 2> "state":"construction",
[junit4] 2> "replicas":{
[junit4] 2> "core_node10":{
[junit4] 2> "core":"testCollection_shard1_0_replica_n9",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":20480,
[junit4] 2> "node_name":"127.0.0.1:10002_solr",
[junit4] 2> "base_url":"http://127.0.0.1:10002/solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":1.9073486328125E-5,
[junit4] 2> "SEARCHER.searcher.numDocs":0},
[junit4] 2> "core_node8":{
[junit4] 2> "core":"testCollection_shard1_0_replica_n7",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":20480,
[junit4] 2> "node_name":"127.0.0.1:10003_solr",
[junit4] 2> "base_url":"http://127.0.0.1:10003/solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":1.9073486328125E-5,
[junit4] 2> "SEARCHER.searcher.numDocs":0}}}
[junit4] 2> 622554 DEBUG (simCloudManagerPool-3671-thread-6) [ ] o.a.s.c.a.s.SimClusterStateProvider ** no leader in testCollection / shard1_0:{
[junit4] 2> "parent":"shard1",
[junit4] 2> "stateTimestamp":"1616694765041868950",
[junit4] 2> "range":"80000000-bfffffff",
[junit4] 2> "state":"construction",
[junit4] 2> "replicas":{
[junit4] 2> "core_node10":{
[junit4] 2> "core":"testCollection_shard1_0_replica_n9",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":20480,
[junit4] 2> "node_name":"127.0.0.1:10002_solr",
[junit4] 2> "base_url":"http://127.0.0.1:10002/solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":1.9073486328125E-5,
[junit4] 2> "SEARCHER.searcher.numDocs":0},
[junit4] 2> "core_node8":{
[junit4] 2> "core":"testCollection_shard1_0_replica_n7",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":20480,
[junit4] 2> "node_name":"127.0.0.1:10003_solr",
[junit4] 2> "base_url":"http://127.0.0.1:10003/solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":1.9073486328125E-5,
[junit4] 2> "SEARCHER.searcher.numDocs":0}}}
[junit4] 2> 622557 DEBUG (simCloudManagerPool-3671-thread-6) [ ] o.a.s.c.a.s.SimClusterStateProvider ** no leader in testCollection / shard1_0:{
[junit4] 2> "parent":"shard1",
[junit4] 2> "stateTimestamp":"1616694765041868950",
[junit4] 2> "range":"80000000-bfffffff",
[junit4] 2> "state":"construction",
[junit4] 2> "replicas":{
[junit4] 2> "core_node10":{
[junit4] 2> "core":"testCollection_shard1_0_replica_n9",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":20480,
[junit4] 2> "node_name":"127.0.0.1:10002_solr",
[junit4] 2> "base_url":"http://127.0.0.1:10002/solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":1.9073486328125E-5,
[junit4] 2> "SEARCHER.searcher.numDocs":0},
[junit4] 2> "core_node8":{
[junit4] 2> "core":"testCollection_shard1_0_replica_n7",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":20480,
[junit4] 2> "node_name":"127.0.0.1:10003_solr",
[junit4] 2> "base_url":"http://127.0.0.1:10003/solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":1.9073486328125E-5,
[junit4] 2> "SEARCHER.searcher.numDocs":0}}}
[junit4] 2> 622559 DEBUG (simCloudManagerPool-3671-thread-6) [ ] o.a.s.c.a.s.SimClusterStateProvider ** no leader in testCollection / shard1_0:{
[junit4] 2> "parent":"shard1",
[junit4] 2> "stateTimestamp":"1616694765041868950",
[junit4] 2> "range":"80000000-bfffffff",
[junit4] 2> "state":"construction",
[junit4] 2> "replicas":{
[junit4] 2> "core_node10":{
[junit4] 2> "core":"testCollection_shard1_0_replica_n9",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":20480,
[junit4] 2> "node_name":"127.0.0.1:10002_solr",
[junit4] 2> "base_url":"http://127.0.0.1:10002/solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":1.9073486328125E-5,
[junit4] 2> "SEARCHER.searcher.numDocs":0},
[junit4] 2> "core_node8":{
[junit4] 2> "core":"testCollection_shard1_0_replica_n7",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":20480,
[junit4] 2> "node_name":"127.0.0.1:10003_solr",
[junit4] 2> "base_url":"http://127.0.0.1:10003/solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":1.9073486328125E-5,
[junit4] 2> "SEARCHER.searcher.numDocs":0}}}
[junit4] 2> 622560 DEBUG (simCloudManagerPool-3671-thread-10) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election (testCollection / shard1_0)
[junit4] 2> 622560 DEBUG (simCloudManagerPool-3671-thread-10) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testCollection / shard1_0 (currentVersion=12): {"core_node10":{}}
[junit4] 2> 622561 DEBUG (ScheduledTrigger-3672-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 622561 DEBUG (simCloudManagerPool-3671-thread-6) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=12
[junit4] 2> 622561 DEBUG (simCloudManagerPool-3671-thread-6) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 12
[junit4] 2> 622562 INFO (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario 7. WaitCollection collection=testCollection&shards=4&withInactive=true&replicas=2&requireLeaders=true
[junit4] 2> 622562 INFO (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario WaitCollection collection=testCollection&shards=4&withInactive=true&replicas=2&requireLeaders=true
[junit4] 2> 622562 DEBUG (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=13
[junit4] 2> 622562 DEBUG (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 13
[junit4] 2> 622563 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 622566 DEBUG (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
[junit4] 2> 622566 DEBUG (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now
[junit4] 2> 622566 DEBUG (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
[junit4] 2> 622566 DEBUG (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor
[junit4] 2> 622566 DEBUG (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
[junit4] 2> 622566 DEBUG (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
[junit4] 2> 622567 INFO (TEST-TestSimScenario.testSplitShard-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testSplitShard
[junit4] 2> 622570 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testAutoAddReplicas
[junit4] 2> 622571 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario 1. CreateCluster numNodes=2
[junit4] 2> 622571 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario CreateCluster numNodes=2
[junit4] 2> 622613 DEBUG (TEST-TestSimScenario.testAutoAddReplicas-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimClusterStateProvider --- new Overseer leader: 127.0.0.1:10004_solr
[junit4] 2> 622615 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
[junit4] 2> 622615 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
[junit4] 2> 622622 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 0
[junit4] 2> 622622 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
[junit4] 2> 622631 DEBUG (TEST-TestSimScenario.testAutoAddReplicas-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=0
[junit4] 2> 622631 DEBUG (TEST-TestSimScenario.testAutoAddReplicas-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 0
[junit4] 2> 622631 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 622631 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario 2. LoadAutoscaling json={'cluster-policy'+:+[{'replica'+:+'<3',+'shard'+:+'#EACH',+'collection'+:+'testCollection','node':'#ANY'}]}&defaultWaitFor=10
[junit4] 2> 622631 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario LoadAutoscaling json={'cluster-policy'+:+[{'replica'+:+'<3',+'shard'+:+'#EACH',+'collection'+:+'testCollection','node':'#ANY'}]}&defaultWaitFor=10
[junit4] 2> 622632 DEBUG (TEST-TestSimScenario.testAutoAddReplicas-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2
[junit4] 2> 622652 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10004_solr, 127.0.0.1:10005_solr]
[junit4] 2> 622664 DEBUG (ScheduledTrigger-3678-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 622686 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- deactivating old nodeLost / nodeAdded markers
[junit4] 2> 622686 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 1
[junit4] 2> 622686 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
[junit4] 2> 622686 DEBUG (ScheduledTrigger-3678-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 622696 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10004_solr, 127.0.0.1:10005_solr]
[junit4] 2> 622696 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- deactivating old nodeLost / nodeAdded markers
[junit4] 2> 622696 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
[junit4] 2> 622697 DEBUG (ScheduledTrigger-3678-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 622697 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario 3. RunSolrRequest path=/admin/collections&action=CREATE&autoAddReplicas=true&name=testCollection&numShards=2&replicationFactor=2&maxShardsPerNode=2
[junit4] 2> 622697 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario RunSolrRequest path=/admin/collections&action=CREATE&autoAddReplicas=true&name=testCollection&numShards=2&replicationFactor=2&maxShardsPerNode=2
[junit4] 2> 622715 DEBUG (simCloudManagerPool-3677-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider -- simCreateCollection testCollection, currentVersion=1
[junit4] 2> 622715 DEBUG (simCloudManagerPool-3677-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=1
[junit4] 2> 622715 DEBUG (simCloudManagerPool-3677-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 1
[junit4] 2> 622718 DEBUG (ScheduledTrigger-3678-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 622725 DEBUG (simCloudManagerPool-3677-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=2
[junit4] 2> 622725 DEBUG (simCloudManagerPool-3677-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 2
[junit4] 2> 622739 DEBUG (ScheduledTrigger-3678-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 622754 DEBUG (simCloudManagerPool-3677-thread-2) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=3
[junit4] 2> 622755 DEBUG (simCloudManagerPool-3677-thread-2) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 3
[junit4] 2> 622755 DEBUG (simCloudManagerPool-3677-thread-2) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election (testCollection / shard1)
[junit4] 2> 622755 DEBUG (simCloudManagerPool-3677-thread-2) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testCollection / shard1 (currentVersion=4): {"core_node1":{}}
[junit4] 2> 622759 DEBUG (ScheduledTrigger-3678-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 622768 DEBUG (simCloudManagerPool-3677-thread-4) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=4
[junit4] 2> 622769 DEBUG (simCloudManagerPool-3677-thread-4) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 4
[junit4] 2> 622769 DEBUG (simCloudManagerPool-3677-thread-4) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election (testCollection / shard2)
[junit4] 2> 622769 DEBUG (simCloudManagerPool-3677-thread-4) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testCollection / shard2 (currentVersion=5): {"core_node3":{}}
[junit4] 2> 622779 DEBUG (simCloudManagerPool-3677-thread-3) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=5
[junit4] 2> 622779 DEBUG (simCloudManagerPool-3677-thread-3) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 5
[junit4] 2> 622779 DEBUG (ScheduledTrigger-3678-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 622779 DEBUG (simCloudManagerPool-3677-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider -- finished createCollection testCollection, currentVersion=6
[junit4] 2> 622779 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario 4. WaitCollection collection=testCollection&shards=2&replicas=2
[junit4] 2> 622779 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario WaitCollection collection=testCollection&shards=2&replicas=2
[junit4] 2> 622789 DEBUG (simCloudManagerPool-3677-thread-5) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=6
[junit4] 2> 622790 DEBUG (simCloudManagerPool-3677-thread-5) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 6
[junit4] 2> 622790 DEBUG (TEST-TestSimScenario.testAutoAddReplicas-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=7
[junit4] 2> 622790 DEBUG (TEST-TestSimScenario.testAutoAddReplicas-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 7
[junit4] 2> 622790 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario 5. SetEventListener trigger=.auto_add_replicas&stage=SUCCEEDED
[junit4] 2> 622790 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario SetEventListener trigger=.auto_add_replicas&stage=SUCCEEDED
[junit4] 2> 622791 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario 6. KillNodes node=${_random_node_}
[junit4] 2> 622791 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario KillNodes node=127.0.0.1:10005_solr
[junit4] 2> 622791 DEBUG (TEST-TestSimScenario.testAutoAddReplicas-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimClusterStateProvider -- created marker: /autoscaling/nodeLost/127.0.0.1:10005_solr
[junit4] 2> 622791 DEBUG (TEST-TestSimScenario.testAutoAddReplicas-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=8
[junit4] 2> 622792 DEBUG (TEST-TestSimScenario.testAutoAddReplicas-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 8
[junit4] 2> 622799 DEBUG (ScheduledTrigger-3678-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 2
[junit4] 2> 622799 DEBUG (ScheduledTrigger-3678-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10005_solr
[junit4] 2> 622809 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario 7. WaitEvent trigger=.auto_add_replicas&wait=60
[junit4] 2> 622809 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario WaitEvent trigger=.auto_add_replicas&wait=60
[junit4] 2> 622809 DEBUG (simCloudManagerPool-3677-thread-7) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=9
[junit4] 2> 622810 DEBUG (simCloudManagerPool-3677-thread-7) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 9
[junit4] 2> 622810 DEBUG (simCloudManagerPool-3677-thread-7) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election (testCollection / shard1)
[junit4] 2> 622810 INFO (simCloudManagerPool-3677-thread-7) [ ] o.a.s.c.ActionThrottle Throttling leader attempts - waiting for 2225ms
[junit4] 2> 622819 DEBUG (ScheduledTrigger-3678-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
[junit4] 2> 622840 DEBUG (ScheduledTrigger-3678-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
[junit4] 2> 622855 DEBUG (simCloudManagerPool-3677-thread-7) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testCollection / shard1 (currentVersion=10): {"core_node2":{}}
[junit4] 2> 622860 DEBUG (ScheduledTrigger-3678-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
[junit4] 2> 623268 DEBUG (ScheduledTrigger-3678-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
[junit4] 2> 623269 DEBUG (ScheduledTrigger-3678-thread-3) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:10005_solr]
[junit4] 2> 623272 DEBUG (ScheduledTrigger-3678-thread-3) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
[junit4] 2> "id":"3f0733b16fc4abT9ef4qfki8d8yiz5de7faali3a",
[junit4] 2> "source":".auto_add_replicas",
[junit4] 2> "eventTime":17740842134389931,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[17740842134389931],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "nodeNames":["127.0.0.1:10005_solr"]}}
[junit4] 2> 623272 DEBUG (ScheduledTrigger-3678-thread-3) [ ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, .scheduled_maintenance]
[junit4] 2> 623273 DEBUG (ScheduledTrigger-3678-thread-3) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=10
[junit4] 2> 623273 DEBUG (ScheduledTrigger-3678-thread-3) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 10
[junit4] 2> 623273 DEBUG (ScheduledTrigger-3678-thread-3) [ ] o.a.s.c.a.SystemLogListener Collection .system missing, skip sending event {
[junit4] 2> "id":"3f0733b16fc4abT9ef4qfki8d8yiz5de7faali3a",
[junit4] 2> "source":".auto_add_replicas",
[junit4] 2> "eventTime":17740842134389931,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[17740842134389931],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":17740865800006281,
[junit4] 2> "nodeNames":["127.0.0.1:10005_solr"]}}
[junit4] 2> 623296 DEBUG (AutoscalingActionExecutor-3679-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
[junit4] 2> "id":"3f0733b16fc4abT9ef4qfki8d8yiz5de7faali3a",
[junit4] 2> "source":".auto_add_replicas",
[junit4] 2> "eventTime":17740842134389931,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[17740842134389931],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":17740865800006281,
[junit4] 2> "nodeNames":["127.0.0.1:10005_solr"]}}
[junit4] 2> 623300 DEBUG (AutoscalingActionExecutor-3679-thread-1) [ ] o.a.s.c.a.ComputePlanAction -- processing event: {
[junit4] 2> "id":"3f0733b16fc4abT9ef4qfki8d8yiz5de7faali3a",
[junit4] 2> "source":".auto_add_replicas",
[junit4] 2> "eventTime":17740842134389931,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[17740842134389931],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":17740865800006281,
[junit4] 2> "nodeNames":["127.0.0.1:10005_solr"]}} with context properties: {BEFORE_ACTION=[auto_add_replicas_plan]}
[junit4] 2> 623310 DEBUG (AutoscalingActionExecutor-3679-thread-1) [ ] o.a.s.c.a.ComputePlanAction -- estimated total max ops=12, effective maxOps=12
[junit4] 2> 623312 DEBUG (AutoscalingActionExecutor-3679-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testCollection&targetNode=127.0.0.1:10004_solr&inPlaceMove=true&replica=core_node1
[junit4] 2> 623314 DEBUG (AutoscalingActionExecutor-3679-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testCollection&targetNode=127.0.0.1:10004_solr&inPlaceMove=true&replica=core_node4
[junit4] 2> 623314 DEBUG (AutoscalingActionExecutor-3679-thread-1) [ ] o.a.s.c.a.ComputePlanAction -- no more operations suggested, stopping after 2 ops...
[junit4] 2> 623314 DEBUG (AutoscalingActionExecutor-3679-thread-1) [ ] o.a.s.c.a.ExecutePlanAction -- processing event: {
[junit4] 2> "id":"3f0733b16fc4abT9ef4qfki8d8yiz5de7faali3a",
[junit4] 2> "source":".auto_add_replicas",
[junit4] 2> "eventTime":17740842134389931,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[17740842134389931],
[junit4] 2> "__compute_diag__":{
[junit4] 2> "noSuggestionsStopAfter":2,
[junit4] 2> "estimatedMaxOps":12,
[junit4] 2> "requestedOperations":-1,
[junit4] 2> "opLimit":12,
[junit4] 2> "maxOperations":12},
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":17740865800006281,
[junit4] 2> "nodeNames":["127.0.0.1:10005_solr"]}} with context properties: {operations=[{
[junit4] 2> "class":"org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica",
[junit4] 2> "method":"GET",
[junit4] 2> "params.action":"MOVEREPLICA",
[junit4] 2> "params.collection":"testCollection",
[junit4] 2> "params.targetNode":"127.0.0.1:10004_solr",
[junit4] 2> "params.inPlaceMove":"true",
[junit4] 2> "params.replica":"core_node1"}, {
[junit4] 2> "class":"org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica",
[junit4] 2> "method":"GET",
[junit4] 2> "params.action":"MOVEREPLICA",
[junit4] 2> "params.collection":"testCollection",
[junit4] 2> "params.targetNode":"127.0.0.1:10004_solr",
[junit4] 2> "params.inPlaceMove":"true",
[junit4] 2> "params.replica":"core_node4"}], AFTER_ACTION=[auto_add_replicas_plan], BEFORE_ACTION=[auto_add_replicas_plan, execute_plan]}
[junit4] 2> 623315 DEBUG (AutoscalingActionExecutor-3679-thread-1) [ ] o.a.s.c.a.ExecutePlanAction Executing operation: action=MOVEREPLICA&collection=testCollection&targetNode=127.0.0.1:10004_solr&inPlaceMove=true&replica=core_node1
[junit4] 2> 623444 DEBUG (simCloudManagerPool-3677-thread-8) [ ] o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node6":{}}
[junit4] 2> 623748 DEBUG (simCloudManagerPool-3677-thread-8) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=11
[junit4] 2> 623749 DEBUG (simCloudManagerPool-3677-thread-8) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 11
[junit4] 2> 623764 DEBUG (AutoscalingActionExecutor-3679-thread-1) [ ] o.a.s.c.a.ExecutePlanAction Executing operation: action=MOVEREPLICA&collection=testCollection&targetNode=127.0.0.1:10004_solr&inPlaceMove=true&replica=core_node4
[junit4] 2> 623771 DEBUG (simCloudManagerPool-3677-thread-11) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=12
[junit4] 2> 623772 DEBUG (simCloudManagerPool-3677-thread-11) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 12
[junit4] 2> 623873 DEBUG (simCloudManagerPool-3677-thread-11) [ ] o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node8":{}}
[junit4] 2> 623983 DEBUG (simCloudManagerPool-3677-thread-11) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=13
[junit4] 2> 623984 DEBUG (simCloudManagerPool-3677-thread-11) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 13
[junit4] 2> 624015 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 624015 DEBUG (TEST-TestSimScenario.testAutoAddReplicas-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
[junit4] 2> 624018 DEBUG (TEST-TestSimScenario.testAutoAddReplicas-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now
[junit4] 2> 624018 DEBUG (TEST-TestSimScenario.testAutoAddReplicas-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
[junit4] 2> 624023 ERROR (AutoscalingActionExecutor-3679-thread-1) [ ] o.a.s.c.a.ExecutePlanAction Unexpected Exception while querying status of requestId=.auto_add_replicas/3f0733b16fc4abT9ef4qfki8d8yiz5de7faali3a/1
[junit4] 2> => java.io.IOException: java.lang.InterruptedException
[junit4] 2> at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.request(SimCloudManager.java:737)
[junit4] 2> java.io.IOException: java.lang.InterruptedException
[junit4] 2> at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.request(SimCloudManager.java:737) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.ExecutePlanAction.waitForTaskToFinish(ExecutePlanAction.java:204) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.ExecutePlanAction.process(ExecutePlanAction.java:120) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:331) ~[java/:?]
[junit4] 2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_281]
[junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_281]
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:218) ~[java/:?]
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_281]
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_281]
[junit4] 2> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_281]
[junit4] 2> Caused by: java.lang.InterruptedException
[junit4] 2> at java.util.concurrent.FutureTask.awaitDone(FutureTask.java:404) ~[?:1.8.0_281]
[junit4] 2> at java.util.concurrent.FutureTask.get(FutureTask.java:204) ~[?:1.8.0_281]
[junit4] 2> at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.request(SimCloudManager.java:735) ~[java/:?]
[junit4] 2> ... 9 more
[junit4] 2> 624023 WARN (AutoscalingActionExecutor-3679-thread-1) [ ] o.a.s.c.a.SystemLogListener Exception sending event. Collection: .system, event: {
[junit4] 2> "id":"3f0733b16fc4abT9ef4qfki8d8yiz5de7faali3a",
[junit4] 2> "source":".auto_add_replicas",
[junit4] 2> "eventTime":17740842134389931,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[17740842134389931],
[junit4] 2> "__compute_diag__":{
[junit4] 2> "noSuggestionsStopAfter":2,
[junit4] 2> "estimatedMaxOps":12,
[junit4] 2> "requestedOperations":-1,
[junit4] 2> "opLimit":12,
[junit4] 2> "maxOperations":12},
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":17740865800006281,
[junit4] 2> "nodeNames":["127.0.0.1:10005_solr"]}}, exception: java.io.IOException: already closed
[junit4] 2> 624023 WARN (AutoscalingActionExecutor-3679-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Exception executing actions
[junit4] 2> => org.apache.solr.cloud.autoscaling.TriggerActionException: Error processing action for trigger event: {
[junit4] 2> "id":"3f0733b16fc4abT9ef4qfki8d8yiz5de7faali3a",
[junit4] 2> org.apache.solr.cloud.autoscaling.TriggerActionException: Error processing action for trigger event: {
[junit4] 2> "id":"3f0733b16fc4abT9ef4qfki8d8yiz5de7faali3a",
[junit4] 2> "source":".auto_add_replicas",
[junit4] 2> "eventTime":17740842134389931,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[17740842134389931],
[junit4] 2> "__compute_diag__":{
[junit4] 2> "noSuggestionsStopAfter":2,
[junit4] 2> "estimatedMaxOps":12,
[junit4] 2> "requestedOperations":-1,
[junit4] 2> "opLimit":12,
[junit4] 2> "maxOperations":12},
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":17740865800006281,
[junit4] 2> "nodeNames":["127.0.0.1:10005_solr"]}}
[junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:334) ~[java/:?]
[junit4] 2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_281]
[junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_281]
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:218) ~[java/:?]
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_281]
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_281]
[junit4] 2> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_281]
[junit4] 2> Caused by: org.apache.solr.common.SolrException: Unexpected exception while processing event: {
[junit4] 2> "id":"3f0733b16fc4abT9ef4qfki8d8yiz5de7faali3a",
[junit4] 2> "source":".auto_add_replicas",
[junit4] 2> "eventTime":17740842134389931,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[17740842134389931],
[junit4] 2> "__compute_diag__":{
[junit4] 2> "noSuggestionsStopAfter":2,
[junit4] 2> "estimatedMaxOps":12,
[junit4] 2> "requestedOperations":-1,
[junit4] 2> "opLimit":12,
[junit4] 2> "maxOperations":12},
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":17740865800006281,
[junit4] 2> "nodeNames":["127.0.0.1:10005_solr"]}}
[junit4] 2> at org.apache.solr.cloud.autoscaling.ExecutePlanAction.process(ExecutePlanAction.java:188) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:331) ~[java/:?]
[junit4] 2> ... 6 more
[junit4] 2> Caused by: org.apache.solr.common.SolrException: Unexpected exception executing operation: action=MOVEREPLICA&async=.auto_add_replicas/3f0733b16fc4abT9ef4qfki8d8yiz5de7faali3a/1&waitForFinalState=true&collection=testCollection&targetNode=127.0.0.1:10004_solr&inPlaceMove=true&replica=core_node4
[junit4] 2> at org.apache.solr.cloud.autoscaling.ExecutePlanAction.process(ExecutePlanAction.java:178) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:331) ~[java/:?]
[junit4] 2> ... 6 more
[junit4] 2> Caused by: java.io.IOException: java.lang.InterruptedException
[junit4] 2> at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.request(SimCloudManager.java:737) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.ExecutePlanAction.waitForTaskToFinish(ExecutePlanAction.java:204) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.ExecutePlanAction.process(ExecutePlanAction.java:120) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:331) ~[java/:?]
[junit4] 2> ... 6 more
[junit4] 2> Caused by: java.lang.InterruptedException
[junit4] 2> at java.util.concurrent.FutureTask.awaitDone(FutureTask.java:404) ~[?:1.8.0_281]
[junit4] 2> at java.util.concurrent.FutureTask.get(FutureTask.java:204) ~[?:1.8.0_281]
[junit4] 2> at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.request(SimCloudManager.java:735) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.ExecutePlanAction.waitForTaskToFinish(ExecutePlanAction.java:204) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.ExecutePlanAction.process(ExecutePlanAction.java:120) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:331) ~[java/:?]
[junit4] 2> ... 6 more
[junit4] 2> 624024 DEBUG (AutoscalingActionExecutor-3679-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 36421 ms for event id=3f0733b16fc4abT9ef4qfki8d8yiz5de7faali3a
[junit4] 2> 624025 DEBUG (TEST-TestSimScenario.testAutoAddReplicas-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor
[junit4] 2> 624025 DEBUG (TEST-TestSimScenario.testAutoAddReplicas-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
[junit4] 2> 624025 DEBUG (TEST-TestSimScenario.testAutoAddReplicas-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
[junit4] 2> 624026 INFO (TEST-TestSimScenario.testAutoAddReplicas-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testAutoAddReplicas
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestSimScenario -Dtests.method=testAutoAddReplicas -Dtests.seed=6AE4FA6F8CF310EA -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=en-NZ -Dtests.timezone=Asia/Almaty -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] ERROR 1.46s J3 | TestSimScenario.testAutoAddReplicas <<<
[junit4] > Throwable #1: java.io.IOException: Timed out waiting for trigger .auto_add_replicas to fire after simulated 60s (real 1200ms).
[junit4] > at __randomizedtesting.SeedInfo.seed([6AE4FA6F8CF310EA:E3440913F8F8DCC3]:0)
[junit4] > at org.apache.solr.cloud.autoscaling.sim.SimScenario$SimWaitListener.wait(SimScenario.java:705)
[junit4] > at org.apache.solr.cloud.autoscaling.sim.SimScenario$WaitEvent.execute(SimScenario.java:771)
[junit4] > at org.apache.solr.cloud.autoscaling.sim.SimScenario.run(SimScenario.java:1119)
[junit4] > at org.apache.solr.cloud.autoscaling.sim.TestSimScenario.testAutoAddReplicas(TestSimScenario.java:57)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 624032 INFO (TEST-TestSimScenario.testSuggestions-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testSuggestions
[junit4] 2> 624032 INFO (TEST-TestSimScenario.testSuggestions-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario 1. CreateCluster numNodes=2
[junit4] 2> 624032 INFO (TEST-TestSimScenario.testSuggestions-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario CreateCluster numNodes=2
[junit4] 2> 624044 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimClusterStateProvider --- new Overseer leader: 127.0.0.1:10006_solr
[junit4] 2> 624045 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
[junit4] 2> 624048 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
[junit4] 2> 624048 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 0
[junit4] 2> 624048 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
[junit4] 2> 624050 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=0
[junit4] 2> 624050 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 0
[junit4] 2> 624050 INFO (TEST-TestSimScenario.testSuggestions-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 624051 INFO (TEST-TestSimScenario.testSuggestions-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario 2. RunSolrRequest path=/admin/collections&action=CREATE&autoAddReplicas=true&name=testCollection&numShards=2&replicationFactor=2&maxShardsPerNode=2
[junit4] 2> 624051 INFO (TEST-TestSimScenario.testSuggestions-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario RunSolrRequest path=/admin/collections&action=CREATE&autoAddReplicas=true&name=testCollection&numShards=2&replicationFactor=2&maxShardsPerNode=2
[junit4] 2> 624051 DEBUG (simCloudManagerPool-3683-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider -- simCreateCollection testCollection, currentVersion=1
[junit4] 2> 624051 DEBUG (simCloudManagerPool-3683-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=1
[junit4] 2> 624067 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10006_solr, 127.0.0.1:10007_solr]
[junit4] 2> 624067 DEBUG (simCloudManagerPool-3683-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 1
[junit4] 2> 624071 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- deactivating old nodeLost / nodeAdded markers
[junit4] 2> 624071 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
[junit4] 2> 624071 DEBUG (ScheduledTrigger-3684-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 624077 DEBUG (simCloudManagerPool-3683-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=2
[junit4] 2> 624077 DEBUG (simCloudManagerPool-3683-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 2
[junit4] 2> 624483 DEBUG (ScheduledTrigger-3684-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 624495 DEBUG (simCloudManagerPool-3683-thread-3) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=3
[junit4] 2> 624495 DEBUG (simCloudManagerPool-3683-thread-3) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 3
[junit4] 2> 624495 DEBUG (simCloudManagerPool-3683-thread-3) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election (testCollection / shard1)
[junit4] 2> 624495 DEBUG (simCloudManagerPool-3683-thread-3) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testCollection / shard1 (currentVersion=4): {"core_node2":{}}
[junit4] 2> 624505 DEBUG (ScheduledTrigger-3684-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 624506 DEBUG (simCloudManagerPool-3683-thread-4) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=4
[junit4] 2> 624506 DEBUG (simCloudManagerPool-3683-thread-4) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 4
[junit4] 2> 624506 DEBUG (simCloudManagerPool-3683-thread-4) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election (testCollection / shard2)
[junit4] 2> 624506 DEBUG (simCloudManagerPool-3683-thread-4) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testCollection / shard2 (currentVersion=5): {"core_node3":{}}
[junit4] 2> 624506 DEBUG (simCloudManagerPool-3683-thread-1) [ ] o.a.s.c.a.s.SimClusterStateProvider -- finished createCollection testCollection, currentVersion=5
[junit4] 2> 624506 INFO (TEST-TestSimScenario.testSuggestions-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario 3. WaitCollection collection=testCollection&shards=2&replicas=2
[junit4] 2> 624506 INFO (TEST-TestSimScenario.testSuggestions-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario WaitCollection collection=testCollection&shards=2&replicas=2
[junit4] 2> 624517 DEBUG (simCloudManagerPool-3683-thread-2) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=5
[junit4] 2> 624517 DEBUG (simCloudManagerPool-3683-thread-2) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 5
[junit4] 2> 624525 DEBUG (ScheduledTrigger-3684-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 624531 DEBUG (simCloudManagerPool-3683-thread-5) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=6
[junit4] 2> 624531 DEBUG (simCloudManagerPool-3683-thread-5) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 6
[junit4] 2> 624532 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=7
[junit4] 2> 624532 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 7
[junit4] 2> 624532 INFO (TEST-TestSimScenario.testSuggestions-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario 4. CtxSet key=myNode&value=${_random_node_}
[junit4] 2> 624532 INFO (TEST-TestSimScenario.testSuggestions-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario CtxSet key=myNode&value=127.0.0.1:10006_solr
[junit4] 2> 624532 INFO (TEST-TestSimScenario.testSuggestions-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario 5. RunSolrRequest path=/admin/collections&action=ADDREPLICA&collection=testCollection&shard=shard1&node=${myNode}
[junit4] 2> 624532 INFO (TEST-TestSimScenario.testSuggestions-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario RunSolrRequest path=/admin/collections&action=ADDREPLICA&collection=testCollection&shard=shard1&node=127.0.0.1:10006_solr
[junit4] 2> 624540 INFO (simCloudManagerPool-3683-thread-6) [ ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:10006_solr for creating new replica of shard shard1 for collection testCollection
[junit4] 2> 624545 DEBUG (ScheduledTrigger-3684-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 624547 INFO (simCloudManagerPool-3683-thread-6) [ ] o.a.s.c.a.c.AddReplicaCmd Returning CreateReplica command.
[junit4] 2> 624557 DEBUG (simCloudManagerPool-3683-thread-6) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=8
[junit4] 2> 624558 DEBUG (simCloudManagerPool-3683-thread-6) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 8
[junit4] 2> 624558 INFO (TEST-TestSimScenario.testSuggestions-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario 6. RunSolrRequest path=/admin/collections&action=ADDREPLICA&collection=testCollection&shard=shard1&node=${myNode}
[junit4] 2> 624558 INFO (TEST-TestSimScenario.testSuggestions-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario RunSolrRequest path=/admin/collections&action=ADDREPLICA&collection=testCollection&shard=shard1&node=127.0.0.1:10006_solr
[junit4] 2> 624567 DEBUG (ScheduledTrigger-3684-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 624568 DEBUG (simCloudManagerPool-3683-thread-7) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=9
[junit4] 2> 624568 DEBUG (simCloudManagerPool-3683-thread-7) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 9
[junit4] 2> 624569 INFO (simCloudManagerPool-3683-thread-7) [ ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:10006_solr for creating new replica of shard shard1 for collection testCollection
[junit4] 2> 624569 INFO (simCloudManagerPool-3683-thread-7) [ ] o.a.s.c.a.c.AddReplicaCmd Returning CreateReplica command.
[junit4] 2> 624579 DEBUG (simCloudManagerPool-3683-thread-7) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=10
[junit4] 2> 624579 DEBUG (simCloudManagerPool-3683-thread-7) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 10
[junit4] 2> 624579 INFO (TEST-TestSimScenario.testSuggestions-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario 7. LoopOp iterations=${iterative}
[junit4] 2> 624579 INFO (TEST-TestSimScenario.testSuggestions-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario LoopOp iterations=0
[junit4] 2> 624579 INFO (TEST-TestSimScenario.testSuggestions-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario 8. LoopOp iterations=${justCalc}
[junit4] 2> 624579 INFO (TEST-TestSimScenario.testSuggestions-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario LoopOp iterations=1
[junit4] 2> 624579 INFO (TEST-TestSimScenario.testSuggestions-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario * iter 1 :
[junit4] 2> 624579 INFO (TEST-TestSimScenario.testSuggestions-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario - CalculateSuggestions )
[junit4] 2> 624582 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=11
[junit4] 2> 624583 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 11
[junit4] 2> 624587 DEBUG (ScheduledTrigger-3684-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 624607 DEBUG (ScheduledTrigger-3684-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 624613 INFO (TEST-TestSimScenario.testSuggestions-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario - 1 suggestions
[junit4] 2> 624613 INFO (TEST-TestSimScenario.testSuggestions-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario - SaveSnapshot path=/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J3/temp/solr.cloud.autoscaling.sim.TestSimScenario_6AE4FA6F8CF310EA-001/tempDir-002/snapshot/0)
[junit4] 2> 624627 DEBUG (ScheduledTrigger-3684-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 624632 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SnapshotDistribStateManager - copying 22 resources from SimDistribStateManager
[junit4] 2> 624648 DEBUG (ScheduledTrigger-3684-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 624668 DEBUG (ScheduledTrigger-3684-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 624688 DEBUG (ScheduledTrigger-3684-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 624708 DEBUG (ScheduledTrigger-3684-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 624715 INFO (TEST-TestSimScenario.testSuggestions-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario 9. Dump redact=true
[junit4] 2> 624715 INFO (TEST-TestSimScenario.testSuggestions-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario Dump redact=true
[junit4] 2> 624716 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SnapshotDistribStateManager - copying 22 resources from SimDistribStateManager
[junit4] 2> 624719 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 624721 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
[junit4] 2> 624721 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now
[junit4] 2> 624721 DEBUG (TEST-TestSimScenario.testSuggestions-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.ScheduledTriggers Awaitin
[...truncated too long message...]
iz5de7faali3e
[junit4] 2> 629197 DEBUG (ScheduledTrigger-3696-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100 and last live nodes: 100
[junit4] 2> 629217 DEBUG (ScheduledTrigger-3696-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100 and last live nodes: 100
[junit4] 2> 629238 DEBUG (ScheduledTrigger-3696-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100 and last live nodes: 100
[junit4] 2> 629258 DEBUG (ScheduledTrigger-3696-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100 and last live nodes: 100
[junit4] 2> 629279 DEBUG (ScheduledTrigger-3696-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100 and last live nodes: 100
[junit4] 2> 629299 DEBUG (ScheduledTrigger-3696-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100 and last live nodes: 100
[junit4] 2> 629319 DEBUG (ScheduledTrigger-3696-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100 and last live nodes: 100
[junit4] 2> 629339 DEBUG (ScheduledTrigger-3696-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100 and last live nodes: 100
[junit4] 2> 629360 DEBUG (ScheduledTrigger-3696-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100 and last live nodes: 100
[junit4] 2> 629380 DEBUG (ScheduledTrigger-3696-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100 and last live nodes: 100
[junit4] 2> 629400 DEBUG (ScheduledTrigger-3696-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100 and last live nodes: 100
[junit4] 2> 629421 DEBUG (ScheduledTrigger-3696-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100 and last live nodes: 100
[junit4] 2> 629441 DEBUG (ScheduledTrigger-3696-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100 and last live nodes: 100
[junit4] 2> 629461 DEBUG (ScheduledTrigger-3696-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100 and last live nodes: 100
[junit4] 2> 629481 DEBUG (ScheduledTrigger-3696-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100 and last live nodes: 100
[junit4] 2> 629502 DEBUG (ScheduledTrigger-3696-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100 and last live nodes: 100
[junit4] 2> 629522 DEBUG (ScheduledTrigger-3696-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100 and last live nodes: 100
[junit4] 2> 629543 DEBUG (ScheduledTrigger-3696-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100 and last live nodes: 100
[junit4] 2> 629563 DEBUG (ScheduledTrigger-3696-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100 and last live nodes: 100
[junit4] 2> 629583 DEBUG (ScheduledTrigger-3696-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100 and last live nodes: 100
[junit4] 2> 629603 INFO (TEST-TestSimScenario.testIndexing-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario 8. WaitEvent trigger=indexSizeTrigger&wait=60
[junit4] 2> 629603 INFO (TEST-TestSimScenario.testIndexing-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario WaitEvent trigger=indexSizeTrigger&wait=60
[junit4] 2> 629603 INFO (TEST-TestSimScenario.testIndexing-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario 9. Assert condition=not_null&key=_trigger_event_indexSizeTrigger
[junit4] 2> 629603 INFO (TEST-TestSimScenario.testIndexing-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario Assert condition=not_null&key=_trigger_event_indexSizeTrigger
[junit4] 2> 629603 DEBUG (ScheduledTrigger-3696-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100 and last live nodes: 100
[junit4] 2> 629605 INFO (TEST-TestSimScenario.testIndexing-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario 10. Assert condition=equals&key=_trigger_event_indexSizeTrigger/eventType&expected=INDEXSIZE
[junit4] 2> 629605 INFO (TEST-TestSimScenario.testIndexing-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario Assert condition=equals&key=_trigger_event_indexSizeTrigger/eventType&expected=INDEXSIZE
[junit4] 2> 629605 INFO (TEST-TestSimScenario.testIndexing-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario 11. Assert condition=equals&key=_trigger_event_indexSizeTrigger/properties/requestedOps[0]/action&expected=SPLITSHARD
[junit4] 2> 629605 INFO (TEST-TestSimScenario.testIndexing-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario Assert condition=equals&key=_trigger_event_indexSizeTrigger/properties/requestedOps[0]/action&expected=SPLITSHARD
[junit4] 2> 629605 INFO (TEST-TestSimScenario.testIndexing-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario 12. WaitCollection collection=testCollection&shards=6&withInactive=true&requireLeaders=false&replicas=2
[junit4] 2> 629605 INFO (TEST-TestSimScenario.testIndexing-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.s.SimScenario WaitCollection collection=testCollection&shards=6&withInactive=true&requireLeaders=false&replicas=2
[junit4] 2> 629606 DEBUG (TEST-TestSimScenario.testIndexing-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
[junit4] 2> 629606 DEBUG (TEST-TestSimScenario.testIndexing-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now
[junit4] 2> 629606 DEBUG (TEST-TestSimScenario.testIndexing-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
[junit4] 2> 629606 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 629609 DEBUG (TEST-TestSimScenario.testIndexing-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor
[junit4] 2> 629610 DEBUG (TEST-TestSimScenario.testIndexing-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
[junit4] 2> 629610 DEBUG (TEST-TestSimScenario.testIndexing-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
[junit4] 2> 629610 INFO (TEST-TestSimScenario.testIndexing-seed#[6AE4FA6F8CF310EA]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testIndexing
[junit4] 2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J3/temp/solr.cloud.autoscaling.sim.TestSimScenario_6AE4FA6F8CF310EA-001
[junit4] 2> NOTE: test params are: codec=CheapBastard, sim=Asserting(RandomSimilarity(queryNorm=true): {}), locale=en-NZ, timezone=Asia/Almaty
[junit4] 2> NOTE: Linux 4.15.0-112-generic amd64/Oracle Corporation 1.8.0_281 (64-bit)/cpus=4,threads=1,free=395277280,total=517996544
[junit4] 2> NOTE: All tests run in this JVM: [RecoveryAfterSoftCommitTest, SolrLogAuditLoggerPluginTest, TestRandomFaceting, TestTermsQParserPlugin, LegacyCloudClusterPropTest, OverseerTaskQueueTest, PrimUtilsTest, SecurityConfHandlerTest, TestSubQueryTransformer, AddReplicaTest, SolrJmxReporterTest, TimeRoutedAliasUpdateProcessorTest, CircularListTest, SplitHandlerTest, BlobRepositoryCloudTest, TestDistributedGrouping, TestManagedStopFilterFactory, DistributedSpellCheckComponentTest, JWTIssuerConfigTest, NoCacheHeaderTest, TestClassicSimilarityFactory, ParsingFieldUpdateProcessorsTest, TestConfigSetsAPIExclusivity, TestIntervalFaceting, TestExportWriter, HdfsRestartWhileUpdatingTest, MetricsConfigTest, ScriptEngineTest, TestLRUStatsCache, TestInPlaceUpdateWithRouteField, TestFiltering, TestZkAclsWithHadoopAuth, ZkSolrClientTest, TestStressReorder, MinimalSchemaTest, TestMinMaxOnMultiValuedField, TestReloadDeadlock, SimpleCollectionCreateDeleteTest, SolrCoreCheckLockOnStartupTest, TestStressCloudBlindAtomicUpdates, HDFSCollectionsAPITest, ZkCollectionPropsCachingTest, MergeStrategyTest, DocumentBuilderTest, ConcurrentCreateCollectionTest, TestSchemaSimilarityResource, TestSolrDeletionPolicy2, SolrTestCaseJ4DeleteCoreTest, PeerSyncWithIndexFingerprintCachingTest, TestEmbeddedSolrServerSchemaAPI, ConnectionManagerTest, TestObjectReleaseTracker, TestComponentsName, TestSha256AuthenticationProvider, TestStressIncrementalBackup, V2StandaloneTest, TestExpandComponent, OutOfBoxZkACLAndCredentialsProvidersTest, ScheduledTriggerTest, TestCaffeineCache, DynamicMapsTest, BJQParserTest, TestPushWriter, CoreAdminRequestStatusTest, UUIDFieldTest, TestCollectionAPIs, TestJsonFacetRefinement, TestSolrCoreParser, ClassificationUpdateProcessorIntegrationTest, TestLegacyNumericUtils, TestSimScenario]
[junit4] Completed [217/934 (1!)] on J3 in 8.50s, 4 tests, 1 error <<< FAILURES!
[...truncated 53290 lines...]