You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2017/12/20 23:49:23 UTC
[JENKINS-EA] Lucene-Solr-master-Linux (64bit/jdk-10-ea+32) - Build
# 21115 - Unstable!
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Linux/21115/
Java: 64bit/jdk-10-ea+32 -XX:-UseCompressedOops -XX:+UseSerialGC
2 tests failed.
FAILED: org.apache.solr.cloud.ShardSplitTest.test
Error Message:
There are still nodes recoverying - waited for 330 seconds
Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 330 seconds
at __randomizedtesting.SeedInfo.seed([472B651A27B2B5E8:CF7F5AC0894ED810]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:185)
at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:140)
at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:135)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:908)
at org.apache.solr.cloud.ShardSplitTest.splitByRouteKeyTest(ShardSplitTest.java:810)
at org.apache.solr.cloud.ShardSplitTest.test(ShardSplitTest.java:102)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:564)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.base/java.lang.Thread.run(Thread.java:844)
FAILED: org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.testBasic
Error Message:
Stack Trace:
java.util.concurrent.TimeoutException
at __randomizedtesting.SeedInfo.seed([472B651A27B2B5E8:ECD1780FF86E33C6]:0)
at org.apache.solr.cloud.autoscaling.sim.SimSolrCloudTestCase.waitForState(SimSolrCloudTestCase.java:188)
at org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.testBasic(TestLargeCluster.java:177)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:564)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.base/java.lang.Thread.run(Thread.java:844)
Build Log:
[...truncated 13578 lines...]
[junit4] Suite: org.apache.solr.cloud.autoscaling.sim.TestLargeCluster
[junit4] 2> 2341661 INFO (SUITE-TestLargeCluster-seed#[472B651A27B2B5E8]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.sim.TestLargeCluster_472B651A27B2B5E8-001/init-core-data-001
[junit4] 2> 2341661 INFO (SUITE-TestLargeCluster-seed#[472B651A27B2B5E8]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 2341662 INFO (SUITE-TestLargeCluster-seed#[472B651A27B2B5E8]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason="", ssl=0.0/0.0, value=0.0/0.0, clientAuth=0.0/0.0)
[junit4] 2> 2341663 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
[junit4] 2> 2341664 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 0
[junit4] 2> 2341664 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10022_solr, 127.0.0.1:10018_solr, 127.0.0.1:10016_solr, 127.0.0.1:10025_solr, 127.0.0.1:10021_solr, 127.0.0.1:10017_solr, 127.0.0.1:10026_solr, 127.0.0.1:10020_solr, 127.0.0.1:10019_solr, 127.0.0.1:10024_solr, 127.0.0.1:10023_solr]
[junit4] 2> 2341664 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion -1
[junit4] 2> 2341664 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 0
[junit4] 2> 2341664 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 2341664 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion 0
[junit4] 2> 2341664 DEBUG (ScheduledTrigger-9047-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 21
[junit4] IGNOR/A 0.00s J1 | TestLargeCluster.testSearchRate
[junit4] > Assumption #1: 'awaitsfix' test group is disabled (@AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-11714"))
[junit4] 2> 2341673 INFO (TEST-TestLargeCluster.testBasic-seed#[472B651A27B2B5E8]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testBasic
[junit4] 2> 2341673 DEBUG (TEST-TestLargeCluster.testBasic-seed#[472B651A27B2B5E8]) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
[junit4] 2> 2341673 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
[junit4] 2> 2341673 INFO (TEST-TestLargeCluster.testBasic-seed#[472B651A27B2B5E8]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
[junit4] 2> 2341674 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
[junit4] 2> 2341674 DEBUG (TEST-TestLargeCluster.testBasic-seed#[472B651A27B2B5E8]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
[junit4] 2> 2341674 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 2341674 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Interrupted
[junit4] 2> java.lang.InterruptedException
[junit4] 2> at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1258)
[junit4] 2> at java.base/java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:317)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:154)
[junit4] 2> at java.base/java.lang.Thread.run(Thread.java:844)
[junit4] 2> 2341674 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
[junit4] 2> 2341674 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2
[junit4] 2> 2341675 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10082_solr, 127.0.0.1:10018_solr, 127.0.0.1:10049_solr, 127.0.0.1:10104_solr, 127.0.0.1:10020_solr, 127.0.0.1:10065_solr, 127.0.0.1:10021_solr, 127.0.0.1:10066_solr, 127.0.0.1:10035_solr, 127.0.0.1:10051_solr, 127.0.0.1:10019_solr, 127.0.0.1:10096_solr, 127.0.0.1:10022_solr, 127.0.0.1:10033_solr, 127.0.0.1:10105_solr, 127.0.0.1:10017_solr, 127.0.0.1:10047_solr, 127.0.0.1:10102_solr, 127.0.0.1:10095_solr, 127.0.0.1:10081_solr, 127.0.0.1:10084_solr, 127.0.0.1:10036_solr, 127.0.0.1:10052_solr, 127.0.0.1:10079_solr, 127.0.0.1:10098_solr, 127.0.0.1:10054_solr, 127.0.0.1:10099_solr, 127.0.0.1:10101_solr, 127.0.0.1:10071_solr, 127.0.0.1:10023_solr, 127.0.0.1:10076_solr, 127.0.0.1:10107_solr, 127.0.0.1:10055_solr, 127.0.0.1:10016_solr, 127.0.0.1:10070_solr, 127.0.0.1:10077_solr, 127.0.0.1:10038_solr, 127.0.0.1:10032_solr, 127.0.0.1:10108_solr, 127.0.0.1:10093_solr, 127.0.0.1:10061_solr, 127.0.0.1:10067_solr, 127.0.0.1:10086_solr, 127.0.0.1:10089_solr, 127.0.0.1:10045_solr, 127.0.0.1:10042_solr, 127.0.0.1:10029_solr, 127.0.0.1:10083_solr, 127.0.0.1:10114_solr, 127.0.0.1:10080_solr, 127.0.0.1:10048_solr, 127.0.0.1:10064_solr, 127.0.0.1:10111_solr, 127.0.0.1:10043_solr, 127.0.0.1:10057_solr, 127.0.0.1:10112_solr, 127.0.0.1:10074_solr, 127.0.0.1:10026_solr, 127.0.0.1:10060_solr, 127.0.0.1:10073_solr, 127.0.0.1:10058_solr, 127.0.0.1:10113_solr, 127.0.0.1:10090_solr, 127.0.0.1:10088_solr, 127.0.0.1:10027_solr, 127.0.0.1:10039_solr, 127.0.0.1:10025_solr, 127.0.0.1:10044_solr, 127.0.0.1:10028_solr, 127.0.0.1:10092_solr, 127.0.0.1:10041_solr, 127.0.0.1:10087_solr, 127.0.0.1:10030_solr, 127.0.0.1:10040_solr, 127.0.0.1:10085_solr, 127.0.0.1:10046_solr, 127.0.0.1:10068_solr, 127.0.0.1:10063_solr, 127.0.0.1:10115_solr, 127.0.0.1:10069_solr, 127.0.0.1:10110_solr, 127.0.0.1:10062_solr, 127.0.0.1:10024_solr, 127.0.0.1:10050_solr, 127.0.0.1:10075_solr, 127.0.0.1:10078_solr, 127.0.0.1:10097_solr, 127.0.0.1:10031_solr, 127.0.0.1:10034_solr, 127.0.0.1:10091_solr, 127.0.0.1:10094_solr, 127.0.0.1:10106_solr, 127.0.0.1:10100_solr, 127.0.0.1:10037_solr, 127.0.0.1:10109_solr, 127.0.0.1:10053_solr, 127.0.0.1:10059_solr, 127.0.0.1:10103_solr, 127.0.0.1:10072_solr, 127.0.0.1:10056_solr]
[junit4] 2> 2341675 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion -1
[junit4] 2> 2341675 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
[junit4] 2> 2341675 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 2341675 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
[junit4] 2> 2341675 DEBUG (ScheduledTrigger-9049-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2341677 DEBUG (simCloudManagerPool-9046-thread-1) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 2341677 DEBUG (simCloudManagerPool-9046-thread-1) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 3
[junit4] 2> 2341677 DEBUG (simCloudManagerPool-9046-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10082_solr, 127.0.0.1:10018_solr, 127.0.0.1:10049_solr, 127.0.0.1:10104_solr, 127.0.0.1:10020_solr, 127.0.0.1:10065_solr, 127.0.0.1:10021_solr, 127.0.0.1:10066_solr, 127.0.0.1:10035_solr, 127.0.0.1:10051_solr, 127.0.0.1:10019_solr, 127.0.0.1:10096_solr, 127.0.0.1:10022_solr, 127.0.0.1:10033_solr, 127.0.0.1:10105_solr, 127.0.0.1:10017_solr, 127.0.0.1:10047_solr, 127.0.0.1:10102_solr, 127.0.0.1:10095_solr, 127.0.0.1:10081_solr, 127.0.0.1:10084_solr, 127.0.0.1:10036_solr, 127.0.0.1:10052_solr, 127.0.0.1:10079_solr, 127.0.0.1:10098_solr, 127.0.0.1:10054_solr, 127.0.0.1:10099_solr, 127.0.0.1:10101_solr, 127.0.0.1:10071_solr, 127.0.0.1:10023_solr, 127.0.0.1:10076_solr, 127.0.0.1:10107_solr, 127.0.0.1:10055_solr, 127.0.0.1:10016_solr, 127.0.0.1:10070_solr, 127.0.0.1:10077_solr, 127.0.0.1:10038_solr, 127.0.0.1:10032_solr, 127.0.0.1:10108_solr, 127.0.0.1:10093_solr, 127.0.0.1:10061_solr, 127.0.0.1:10067_solr, 127.0.0.1:10086_solr, 127.0.0.1:10089_solr, 127.0.0.1:10045_solr, 127.0.0.1:10042_solr, 127.0.0.1:10029_solr, 127.0.0.1:10083_solr, 127.0.0.1:10114_solr, 127.0.0.1:10080_solr, 127.0.0.1:10048_solr, 127.0.0.1:10064_solr, 127.0.0.1:10111_solr, 127.0.0.1:10043_solr, 127.0.0.1:10057_solr, 127.0.0.1:10112_solr, 127.0.0.1:10074_solr, 127.0.0.1:10026_solr, 127.0.0.1:10060_solr, 127.0.0.1:10073_solr, 127.0.0.1:10058_solr, 127.0.0.1:10113_solr, 127.0.0.1:10090_solr, 127.0.0.1:10088_solr, 127.0.0.1:10027_solr, 127.0.0.1:10039_solr, 127.0.0.1:10025_solr, 127.0.0.1:10044_solr, 127.0.0.1:10028_solr, 127.0.0.1:10092_solr, 127.0.0.1:10041_solr, 127.0.0.1:10087_solr, 127.0.0.1:10030_solr, 127.0.0.1:10040_solr, 127.0.0.1:10085_solr, 127.0.0.1:10046_solr, 127.0.0.1:10068_solr, 127.0.0.1:10063_solr, 127.0.0.1:10115_solr, 127.0.0.1:10069_solr, 127.0.0.1:10110_solr, 127.0.0.1:10062_solr, 127.0.0.1:10024_solr, 127.0.0.1:10050_solr, 127.0.0.1:10075_solr, 127.0.0.1:10078_solr, 127.0.0.1:10097_solr, 127.0.0.1:10031_solr, 127.0.0.1:10034_solr, 127.0.0.1:10091_solr, 127.0.0.1:10094_solr, 127.0.0.1:10106_solr, 127.0.0.1:10100_solr, 127.0.0.1:10037_solr, 127.0.0.1:10109_solr, 127.0.0.1:10053_solr, 127.0.0.1:10059_solr, 127.0.0.1:10103_solr, 127.0.0.1:10072_solr, 127.0.0.1:10056_solr]
[junit4] 2> 2341678 DEBUG (simCloudManagerPool-9046-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10082_solr, 127.0.0.1:10018_solr, 127.0.0.1:10049_solr, 127.0.0.1:10104_solr, 127.0.0.1:10020_solr, 127.0.0.1:10065_solr, 127.0.0.1:10021_solr, 127.0.0.1:10066_solr, 127.0.0.1:10035_solr, 127.0.0.1:10051_solr, 127.0.0.1:10019_solr, 127.0.0.1:10096_solr, 127.0.0.1:10022_solr, 127.0.0.1:10033_solr, 127.0.0.1:10105_solr, 127.0.0.1:10017_solr, 127.0.0.1:10047_solr, 127.0.0.1:10102_solr, 127.0.0.1:10095_solr, 127.0.0.1:10081_solr, 127.0.0.1:10084_solr, 127.0.0.1:10036_solr, 127.0.0.1:10052_solr, 127.0.0.1:10079_solr, 127.0.0.1:10098_solr, 127.0.0.1:10054_solr, 127.0.0.1:10099_solr, 127.0.0.1:10101_solr, 127.0.0.1:10071_solr, 127.0.0.1:10023_solr, 127.0.0.1:10076_solr, 127.0.0.1:10107_solr, 127.0.0.1:10055_solr, 127.0.0.1:10016_solr, 127.0.0.1:10070_solr, 127.0.0.1:10077_solr, 127.0.0.1:10038_solr, 127.0.0.1:10032_solr, 127.0.0.1:10108_solr, 127.0.0.1:10093_solr, 127.0.0.1:10061_solr, 127.0.0.1:10067_solr, 127.0.0.1:10086_solr, 127.0.0.1:10089_solr, 127.0.0.1:10045_solr, 127.0.0.1:10042_solr, 127.0.0.1:10029_solr, 127.0.0.1:10083_solr, 127.0.0.1:10114_solr, 127.0.0.1:10080_solr, 127.0.0.1:10048_solr, 127.0.0.1:10064_solr, 127.0.0.1:10111_solr, 127.0.0.1:10043_solr, 127.0.0.1:10057_solr, 127.0.0.1:10112_solr, 127.0.0.1:10074_solr, 127.0.0.1:10026_solr, 127.0.0.1:10060_solr, 127.0.0.1:10073_solr, 127.0.0.1:10058_solr, 127.0.0.1:10113_solr, 127.0.0.1:10090_solr, 127.0.0.1:10088_solr, 127.0.0.1:10027_solr, 127.0.0.1:10039_solr, 127.0.0.1:10025_solr, 127.0.0.1:10044_solr, 127.0.0.1:10028_solr, 127.0.0.1:10092_solr, 127.0.0.1:10041_solr, 127.0.0.1:10087_solr, 127.0.0.1:10030_solr, 127.0.0.1:10040_solr, 127.0.0.1:10085_solr, 127.0.0.1:10046_solr, 127.0.0.1:10068_solr, 127.0.0.1:10063_solr, 127.0.0.1:10115_solr, 127.0.0.1:10069_solr, 127.0.0.1:10110_solr, 127.0.0.1:10062_solr, 127.0.0.1:10024_solr, 127.0.0.1:10050_solr, 127.0.0.1:10075_solr, 127.0.0.1:10078_solr, 127.0.0.1:10097_solr, 127.0.0.1:10031_solr, 127.0.0.1:10034_solr, 127.0.0.1:10091_solr, 127.0.0.1:10094_solr, 127.0.0.1:10106_solr, 127.0.0.1:10100_solr, 127.0.0.1:10037_solr, 127.0.0.1:10109_solr, 127.0.0.1:10053_solr, 127.0.0.1:10059_solr, 127.0.0.1:10103_solr, 127.0.0.1:10072_solr, 127.0.0.1:10056_solr]
[junit4] 2> 2341678 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
[junit4] 2> 2341679 DEBUG (ScheduledTrigger-9049-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2341679 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 2341679 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
[junit4] 2> 2341681 DEBUG (simCloudManagerPool-9046-thread-2) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 2341681 DEBUG (simCloudManagerPool-9046-thread-2) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 4
[junit4] 2> 2341681 DEBUG (simCloudManagerPool-9046-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10082_solr, 127.0.0.1:10018_solr, 127.0.0.1:10049_solr, 127.0.0.1:10104_solr, 127.0.0.1:10020_solr, 127.0.0.1:10065_solr, 127.0.0.1:10021_solr, 127.0.0.1:10066_solr, 127.0.0.1:10035_solr, 127.0.0.1:10051_solr, 127.0.0.1:10019_solr, 127.0.0.1:10096_solr, 127.0.0.1:10022_solr, 127.0.0.1:10033_solr, 127.0.0.1:10105_solr, 127.0.0.1:10017_solr, 127.0.0.1:10047_solr, 127.0.0.1:10102_solr, 127.0.0.1:10095_solr, 127.0.0.1:10081_solr, 127.0.0.1:10084_solr, 127.0.0.1:10036_solr, 127.0.0.1:10052_solr, 127.0.0.1:10079_solr, 127.0.0.1:10098_solr, 127.0.0.1:10054_solr, 127.0.0.1:10099_solr, 127.0.0.1:10101_solr, 127.0.0.1:10071_solr, 127.0.0.1:10023_solr, 127.0.0.1:10076_solr, 127.0.0.1:10107_solr, 127.0.0.1:10055_solr, 127.0.0.1:10016_solr, 127.0.0.1:10070_solr, 127.0.0.1:10077_solr, 127.0.0.1:10038_solr, 127.0.0.1:10032_solr, 127.0.0.1:10108_solr, 127.0.0.1:10093_solr, 127.0.0.1:10061_solr, 127.0.0.1:10067_solr, 127.0.0.1:10086_solr, 127.0.0.1:10089_solr, 127.0.0.1:10045_solr, 127.0.0.1:10042_solr, 127.0.0.1:10029_solr, 127.0.0.1:10083_solr, 127.0.0.1:10114_solr, 127.0.0.1:10080_solr, 127.0.0.1:10048_solr, 127.0.0.1:10064_solr, 127.0.0.1:10111_solr, 127.0.0.1:10043_solr, 127.0.0.1:10057_solr, 127.0.0.1:10112_solr, 127.0.0.1:10074_solr, 127.0.0.1:10026_solr, 127.0.0.1:10060_solr, 127.0.0.1:10073_solr, 127.0.0.1:10058_solr, 127.0.0.1:10113_solr, 127.0.0.1:10090_solr, 127.0.0.1:10088_solr, 127.0.0.1:10027_solr, 127.0.0.1:10039_solr, 127.0.0.1:10025_solr, 127.0.0.1:10044_solr, 127.0.0.1:10028_solr, 127.0.0.1:10092_solr, 127.0.0.1:10041_solr, 127.0.0.1:10087_solr, 127.0.0.1:10030_solr, 127.0.0.1:10040_solr, 127.0.0.1:10085_solr, 127.0.0.1:10046_solr, 127.0.0.1:10068_solr, 127.0.0.1:10063_solr, 127.0.0.1:10115_solr, 127.0.0.1:10069_solr, 127.0.0.1:10110_solr, 127.0.0.1:10062_solr, 127.0.0.1:10024_solr, 127.0.0.1:10050_solr, 127.0.0.1:10075_solr, 127.0.0.1:10078_solr, 127.0.0.1:10097_solr, 127.0.0.1:10031_solr, 127.0.0.1:10034_solr, 127.0.0.1:10091_solr, 127.0.0.1:10094_solr, 127.0.0.1:10106_solr, 127.0.0.1:10100_solr, 127.0.0.1:10037_solr, 127.0.0.1:10109_solr, 127.0.0.1:10053_solr, 127.0.0.1:10059_solr, 127.0.0.1:10103_solr, 127.0.0.1:10072_solr, 127.0.0.1:10056_solr]
[junit4] 2> 2341681 DEBUG (simCloudManagerPool-9046-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10082_solr, 127.0.0.1:10018_solr, 127.0.0.1:10049_solr, 127.0.0.1:10104_solr, 127.0.0.1:10020_solr, 127.0.0.1:10065_solr, 127.0.0.1:10021_solr, 127.0.0.1:10066_solr, 127.0.0.1:10035_solr, 127.0.0.1:10051_solr, 127.0.0.1:10019_solr, 127.0.0.1:10096_solr, 127.0.0.1:10022_solr, 127.0.0.1:10033_solr, 127.0.0.1:10105_solr, 127.0.0.1:10017_solr, 127.0.0.1:10047_solr, 127.0.0.1:10102_solr, 127.0.0.1:10095_solr, 127.0.0.1:10081_solr, 127.0.0.1:10084_solr, 127.0.0.1:10036_solr, 127.0.0.1:10052_solr, 127.0.0.1:10079_solr, 127.0.0.1:10098_solr, 127.0.0.1:10054_solr, 127.0.0.1:10099_solr, 127.0.0.1:10101_solr, 127.0.0.1:10071_solr, 127.0.0.1:10023_solr, 127.0.0.1:10076_solr, 127.0.0.1:10107_solr, 127.0.0.1:10055_solr, 127.0.0.1:10016_solr, 127.0.0.1:10070_solr, 127.0.0.1:10077_solr, 127.0.0.1:10038_solr, 127.0.0.1:10032_solr, 127.0.0.1:10108_solr, 127.0.0.1:10093_solr, 127.0.0.1:10061_solr, 127.0.0.1:10067_solr, 127.0.0.1:10086_solr, 127.0.0.1:10089_solr, 127.0.0.1:10045_solr, 127.0.0.1:10042_solr, 127.0.0.1:10029_solr, 127.0.0.1:10083_solr, 127.0.0.1:10114_solr, 127.0.0.1:10080_solr, 127.0.0.1:10048_solr, 127.0.0.1:10064_solr, 127.0.0.1:10111_solr, 127.0.0.1:10043_solr, 127.0.0.1:10057_solr, 127.0.0.1:10112_solr, 127.0.0.1:10074_solr, 127.0.0.1:10026_solr, 127.0.0.1:10060_solr, 127.0.0.1:10073_solr, 127.0.0.1:10058_solr, 127.0.0.1:10113_solr, 127.0.0.1:10090_solr, 127.0.0.1:10088_solr, 127.0.0.1:10027_solr, 127.0.0.1:10039_solr, 127.0.0.1:10025_solr, 127.0.0.1:10044_solr, 127.0.0.1:10028_solr, 127.0.0.1:10092_solr, 127.0.0.1:10041_solr, 127.0.0.1:10087_solr, 127.0.0.1:10030_solr, 127.0.0.1:10040_solr, 127.0.0.1:10085_solr, 127.0.0.1:10046_solr, 127.0.0.1:10068_solr, 127.0.0.1:10063_solr, 127.0.0.1:10115_solr, 127.0.0.1:10069_solr, 127.0.0.1:10110_solr, 127.0.0.1:10062_solr, 127.0.0.1:10024_solr, 127.0.0.1:10050_solr, 127.0.0.1:10075_solr, 127.0.0.1:10078_solr, 127.0.0.1:10097_solr, 127.0.0.1:10031_solr, 127.0.0.1:10034_solr, 127.0.0.1:10091_solr, 127.0.0.1:10094_solr, 127.0.0.1:10106_solr, 127.0.0.1:10100_solr, 127.0.0.1:10037_solr, 127.0.0.1:10109_solr, 127.0.0.1:10053_solr, 127.0.0.1:10059_solr, 127.0.0.1:10103_solr, 127.0.0.1:10072_solr, 127.0.0.1:10056_solr]
[junit4] 2> 2341682 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 4
[junit4] 2> 2341682 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 2341683 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion 4
[junit4] 2> 2341696 DEBUG (ScheduledTrigger-9049-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2341699 DEBUG (ScheduledTrigger-9049-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2341716 DEBUG (ScheduledTrigger-9049-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2341719 DEBUG (ScheduledTrigger-9049-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2341736 DEBUG (ScheduledTrigger-9049-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2341740 DEBUG (ScheduledTrigger-9049-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2341757 DEBUG (ScheduledTrigger-9049-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2341760 DEBUG (ScheduledTrigger-9049-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2341777 DEBUG (ScheduledTrigger-9049-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2341780 DEBUG (ScheduledTrigger-9049-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2341783 DEBUG (simCloudManagerPool-9046-thread-3) [ ] o.a.s.c.a.s.SimCloudManager Invoking Collection Action :create with params ?action=CREATE&name=testBasic&collection.configName=conf&createNodeSet=127.0.0.1%3A10112_solr%2C127.0.0.1%3A10028_solr%2C127.0.0.1%3A10061_solr%2C127.0.0.1%3A10043_solr%2C127.0.0.1%3A10095_solr%2C127.0.0.1%3A10060_solr%2C127.0.0.1%3A10064_solr%2C127.0.0.1%3A10083_solr%2C127.0.0.1%3A10022_solr%2C127.0.0.1%3A10087_solr%2C127.0.0.1%3A10018_solr%2C127.0.0.1%3A10081_solr%2C127.0.0.1%3A10070_solr%2C127.0.0.1%3A10039_solr%2C127.0.0.1%3A10093_solr%2C127.0.0.1%3A10042_solr%2C127.0.0.1%3A10099_solr%2C127.0.0.1%3A10080_solr%2C127.0.0.1%3A10054_solr%2C127.0.0.1%3A10023_solr%2C127.0.0.1%3A10055_solr%2C127.0.0.1%3A10045_solr%2C127.0.0.1%3A10029_solr%2C127.0.0.1%3A10047_solr%2C127.0.0.1%3A10058_solr%2C127.0.0.1%3A10098_solr%2C127.0.0.1%3A10032_solr%2C127.0.0.1%3A10027_solr%2C127.0.0.1%3A10041_solr%2C127.0.0.1%3A10040_solr%2C127.0.0.1%3A10065_solr%2C127.0.0.1%3A10089_solr%2C127.0.0.1%3A10030_solr%2C127.0.0.1%3A10021_solr%2C127.0.0.1%3A10092_solr%2C127.0.0.1%3A10085_solr%2C127.0.0.1%3A10033_solr%2C127.0.0.1%3A10038_solr%2C127.0.0.1%3A10017_solr%2C127.0.0.1%3A10113_solr%2C127.0.0.1%3A10077_solr%2C127.0.0.1%3A10066_solr%2C127.0.0.1%3A10048_solr%2C127.0.0.1%3A10071_solr%2C127.0.0.1%3A10114_solr%2C127.0.0.1%3A10086_solr%2C127.0.0.1%3A10102_solr%2C127.0.0.1%3A10105_solr%2C127.0.0.1%3A10067_solr%2C127.0.0.1%3A10036_solr%2C127.0.0.1%3A10111_solr%2C127.0.0.1%3A10084_solr%2C127.0.0.1%3A10076_solr%2C127.0.0.1%3A10051_solr%2C127.0.0.1%3A10026_solr%2C127.0.0.1%3A10074_solr%2C127.0.0.1%3A10088_solr%2C127.0.0.1%3A10025_solr%2C127.0.0.1%3A10073_solr%2C127.0.0.1%3A10016_solr%2C127.0.0.1%3A10082_solr%2C127.0.0.1%3A10035_solr%2C127.0.0.1%3A10096_solr%2C127.0.0.1%3A10090_solr%2C127.0.0.1%3A10108_solr%2C127.0.0.1%3A10057_solr%2C127.0.0.1%3A10104_solr%2C127.0.0.1%3A10101_solr%2C127.0.0.1%3A10079_solr%2C127.0.0.1%3A10019_solr%2C127.0.0.1%3A10020_solr%2C127.0.0.1%3A10046_solr%2C127.0.0.1%3A10049_solr%2C127.0.0.1%3A10107_solr%2C127.0.0.1%3A10044_solr%2C127.0.0.1%3A10052_solr&numShards=5&maxShardsPerNode=1&replicationFactor=5&nrtReplicas=5&pullReplicas=5&tlogReplicas=5
[junit4] 2> 2341797 DEBUG (ScheduledTrigger-9049-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2341800 DEBUG (ScheduledTrigger-9049-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2341817 DEBUG (ScheduledTrigger-9049-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2341820 DEBUG (ScheduledTrigger-9049-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2341837 DEBUG (ScheduledTrigger-9049-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2341841 DEBUG (ScheduledTrigger-9049-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2341858 DEBUG (ScheduledTrigger-9049-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2341861 DEBUG (ScheduledTrigger-9049-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2341878 DEBUG (ScheduledTrigger-9049-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2341881 DEBUG (ScheduledTrigger-9049-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2341899 DEBUG (ScheduledTrigger-9049-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2341902 DEBUG (ScheduledTrigger-9049-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2341919 DEBUG (ScheduledTrigger-9049-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2341922 DEBUG (ScheduledTrigger-9049-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2341939 DEBUG (ScheduledTrigger-9049-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2341942 DEBUG (ScheduledTrigger-9049-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2341960 DEBUG (ScheduledTrigger-9049-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2341963 DEBUG (ScheduledTrigger-9049-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2341980 DEBUG (ScheduledTrigger-9049-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2341983 DEBUG (ScheduledTrigger-9049-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2342000 DEBUG (ScheduledTrigger-9049-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2342003 DEBUG (ScheduledTrigger-9049-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2342020 DEBUG (ScheduledTrigger-9049-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2342024 DEBUG (ScheduledTrigger-9049-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2342041 DEBUG (ScheduledTrigger-9049-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2342044 DEBUG (ScheduledTrigger-9049-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2342061 DEBUG (ScheduledTrigger-9049-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2342064 DEBUG (ScheduledTrigger-9049-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2342081 DEBUG (ScheduledTrigger-9049-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2342085 DEBUG (ScheduledTrigger-9049-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2342102 DEBUG (ScheduledTrigger-9049-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2342105 DEBUG (ScheduledTrigger-9049-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2342122 DEBUG (ScheduledTrigger-9049-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2342125 DEBUG (ScheduledTrigger-9049-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2342142 DEBUG (ScheduledTrigger-9049-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2342146 DEBUG (ScheduledTrigger-9049-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2342162 DEBUG (ScheduledTrigger-9049-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2342166 DEBUG (ScheduledTrigger-9049-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2342183 DEBUG (ScheduledTrigger-9049-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2342186 DEBUG (ScheduledTrigger-9049-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2342203 DEBUG (ScheduledTrigger-9049-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2342207 DEBUG (ScheduledTrigger-9049-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2342223 DEBUG (ScheduledTrigger-9049-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2342227 DEBUG (ScheduledTrigger-9049-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2342244 DEBUG (ScheduledTrigger-9049-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2342247 DEBUG (ScheduledTrigger-9049-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2342264 DEBUG (ScheduledTrigger-9049-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2342268 DEBUG (ScheduledTrigger-9049-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2342284 DEBUG (ScheduledTrigger-9049-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2342288 DEBUG (ScheduledTrigger-9049-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2342305 DEBUG (ScheduledTrigger-9049-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2342308 DEBUG (ScheduledTrigger-9049-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2342325 DEBUG (ScheduledTrigger-9049-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2342328 DEBUG (ScheduledTrigger-9049-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2342345 DEBUG (ScheduledTrigger-9049-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2342349 DEBUG (ScheduledTrigger-9049-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2342365 DEBUG (ScheduledTrigger-9049-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2342369 DEBUG (ScheduledTrigger-9049-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2342386 DEBUG (ScheduledTrigger-9049-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2342389 DEBUG (ScheduledTrigger-9049-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2342406 DEBUG (ScheduledTrigger-9049-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2342410 DEBUG (ScheduledTrigger-9049-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2342426 DEBUG (ScheduledTrigger-9049-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2342430 DEBUG (ScheduledTrigger-9049-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2342447 DEBUG (ScheduledTrigger-9049-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2342450 DEBUG (ScheduledTrigger-9049-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2342467 DEBUG (ScheduledTrigger-9049-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2342471 DEBUG (ScheduledTrigger-9049-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2342487 DEBUG (ScheduledTrigger-9049-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2342491 DEBUG (ScheduledTrigger-9049-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2342508 DEBUG (ScheduledTrigger-9049-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2342511 DEBUG (ScheduledTrigger-9049-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2342528 DEBUG (ScheduledTrigger-9049-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2342532 DEBUG (ScheduledTrigger-9049-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2342548 DEBUG (ScheduledTrigger-9049-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2342552 DEBUG (ScheduledTrigger-9049-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2342555 INFO (TEST-TestLargeCluster.testBasic-seed#[472B651A27B2B5E8]) [ ] o.a.s.c.a.s.TestLargeCluster Ready after 37937ms
[junit4] 2> 2342556 INFO (simCloudManagerPool-9046-thread-80) [ ] o.a.s.c.ActionThrottle The last leader attempt started 37970ms ago.
[junit4] 2> 2342556 INFO (simCloudManagerPool-9046-thread-81) [ ] o.a.s.c.ActionThrottle The last leader attempt started 15ms ago.
[junit4] 2> 2342556 INFO (simCloudManagerPool-9046-thread-81) [ ] o.a.s.c.ActionThrottle Throttling leader attempts - waiting for 4984ms
[junit4] 2> 2342556 INFO (simCloudManagerPool-9046-thread-82) [ ] o.a.s.c.ActionThrottle The last leader attempt started 25ms ago.
[junit4] 2> 2342556 INFO (simCloudManagerPool-9046-thread-82) [ ] o.a.s.c.ActionThrottle Throttling leader attempts - waiting for 4974ms
[junit4] 2> 2342557 INFO (simCloudManagerPool-9046-thread-83) [ ] o.a.s.c.ActionThrottle The last leader attempt started 35ms ago.
[junit4] 2> 2342557 INFO (simCloudManagerPool-9046-thread-83) [ ] o.a.s.c.ActionThrottle Throttling leader attempts - waiting for 4964ms
[junit4] 2> 2342557 INFO (simCloudManagerPool-9046-thread-84) [ ] o.a.s.c.ActionThrottle The last leader attempt started 45ms ago.
[junit4] 2> 2342557 INFO (simCloudManagerPool-9046-thread-84) [ ] o.a.s.c.ActionThrottle Throttling leader attempts - waiting for 4954ms
[junit4] 2> 2342557 INFO (simCloudManagerPool-9046-thread-85) [ ] o.a.s.c.ActionThrottle The last leader attempt started 55ms ago.
[junit4] 2> 2342557 INFO (simCloudManagerPool-9046-thread-85) [ ] o.a.s.c.ActionThrottle Throttling leader attempts - waiting for 4944ms
[junit4] 2> 2342557 INFO (simCloudManagerPool-9046-thread-86) [ ] o.a.s.c.ActionThrottle The last leader attempt started 60ms ago.
[junit4] 2> 2342557 INFO (simCloudManagerPool-9046-thread-86) [ ] o.a.s.c.ActionThrottle Throttling leader attempts - waiting for 4939ms
[junit4] 2> 2342557 INFO (simCloudManagerPool-9046-thread-87) [ ] o.a.s.c.ActionThrottle The last leader attempt started 75ms ago.
[junit4] 2> 2342558 INFO (simCloudManagerPool-9046-thread-87) [ ] o.a.s.c.ActionThrottle Throttling leader attempts - waiting for 4924ms
[junit4] 2> 2342560 DEBUG (simCloudManagerPool-9046-thread-80) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testBasic / shard2: {"core_node18":{
[junit4] 2> "node_name":"127.0.0.1:10027_solr",
[junit4] 2> "core":"testBasic_shard2_replica_n18",
[junit4] 2> "leader":"true",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInBytes":123450000,
[junit4] 2> "shard":"shard2",
[junit4] 2> "collection":"testBasic"}}
[junit4] 2> 2342561 DEBUG (simCloudManagerPool-9046-thread-80) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testBasic / shard3: {"core_node39":{
[junit4] 2> "node_name":"127.0.0.1:10113_solr",
[junit4] 2> "core":"testBasic_shard3_replica_t39",
[junit4] 2> "leader":"true",
[junit4] 2> "state":"active",
[junit4] 2> "type":"TLOG",
[junit4] 2> "INDEX.sizeInBytes":123450000,
[junit4] 2> "shard":"shard3",
[junit4] 2> "collection":"testBasic"}}
[junit4] 2> 2342561 DEBUG (simCloudManagerPool-9046-thread-80) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testBasic / shard4: {"core_node54":{
[junit4] 2> "node_name":"127.0.0.1:10054_solr",
[junit4] 2> "core":"testBasic_shard4_replica_t54",
[junit4] 2> "leader":"true",
[junit4] 2> "state":"active",
[junit4] 2> "type":"TLOG",
[junit4] 2> "INDEX.sizeInBytes":123450000,
[junit4] 2> "shard":"shard4",
[junit4] 2> "collection":"testBasic"}}
[junit4] 2> 2342561 DEBUG (simCloudManagerPool-9046-thread-80) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testBasic / shard5: {"core_node65":{
[junit4] 2> "node_name":"127.0.0.1:10052_solr",
[junit4] 2> "core":"testBasic_shard5_replica_n65",
[junit4] 2> "leader":"true",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInBytes":123450000,
[junit4] 2> "shard":"shard5",
[junit4] 2> "collection":"testBasic"}}
[junit4] 2> 2342561 DEBUG (simCloudManagerPool-9046-thread-80) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testBasic / shard1: {"core_node1":{
[junit4] 2> "node_name":"127.0.0.1:10021_solr",
[junit4] 2> "core":"testBasic_shard1_replica_n1",
[junit4] 2> "leader":"true",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInBytes":123450000,
[junit4] 2> "shard":"shard1",
[junit4] 2> "collection":"testBasic"}}
[junit4] 2> 2342568 DEBUG (ScheduledTrigger-9049-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 92
[junit4] 2> 2342569 DEBUG (ScheduledTrigger-9049-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10095_solr
[junit4] 2> 2342569 DEBUG (ScheduledTrigger-9049-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10061_solr
[junit4] 2> 2342569 DEBUG (ScheduledTrigger-9049-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10083_solr
[junit4] 2> 2342569 DEBUG (ScheduledTrigger-9049-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10064_solr
[junit4] 2> 2342569 DEBUG (ScheduledTrigger-9049-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10043_solr
[junit4] 2> 2342569 DEBUG (ScheduledTrigger-9049-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10112_solr
[junit4] 2> 2342569 DEBUG (ScheduledTrigger-9049-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10060_solr
[junit4] 2> 2342569 DEBUG (ScheduledTrigger-9049-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10028_solr
[junit4] 2> 2342572 DEBUG (ScheduledTrigger-9049-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 92
[junit4] 2> 2342572 DEBUG (ScheduledTrigger-9049-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10095_solr
[junit4] 2> 2342572 DEBUG (ScheduledTrigger-9049-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10061_solr
[junit4] 2> 2342572 DEBUG (ScheduledTrigger-9049-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10083_solr
[junit4] 2> 2342572 DEBUG (ScheduledTrigger-9049-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10064_solr
[junit4] 2> 2342572 DEBUG (ScheduledTrigger-9049-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10043_solr
[junit4] 2> 2342572 DEBUG (ScheduledTrigger-9049-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10112_solr
[junit4] 2> 2342572 DEBUG (ScheduledTrigger-9049-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10060_solr
[junit4] 2> 2342572 DEBUG (ScheduledTrigger-9049-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10028_solr
[junit4] 2> 2342589 DEBUG (ScheduledTrigger-9049-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 92
[junit4] 2> 2342592 DEBUG (ScheduledTrigger-9049-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 92
[junit4] 2> 2342609 DEBUG (ScheduledTrigger-9049-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 92
[junit4] 2> 2342613 DEBUG (ScheduledTrigger-9049-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 92
[junit4] 2> 2342629 DEBUG (ScheduledTrigger-9049-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 92
[junit4] 2> 2342633 DEBUG (ScheduledTrigger-9049-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 92
[junit4] 2> 2342633 DEBUG (ScheduledTrigger-9049-thread-3) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:10043_solr, 127.0.0.1:10061_solr, 127.0.0.1:10112_solr, 127.0.0.1:10095_solr, 127.0.0.1:10083_solr, 127.0.0.1:10028_solr, 127.0.0.1:10060_solr, 127.0.0.1:10064_solr]
[junit4] 2> 2342633 DEBUG (ScheduledTrigger-9049-thread-3) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
[junit4] 2> "id":"5c1193e633941Tnu8ynavyq8o6u2intom037rr",
[junit4] 2> "source":"node_lost_trigger",
[junit4] 2> "eventTime":1619689048586561,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 1619689048586561,
[junit4] 2> 1619689044798861,
[junit4] 2> 1619689049846161,
[junit4] 2> 1619689043423561,
[junit4] 2> 1619689046072911,
[junit4] 2> 1619689052384461,
[junit4] 2> 1619689051113211,
[junit4] 2> 1619689047331511],
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10043_solr",
[junit4] 2> "127.0.0.1:10061_solr",
[junit4] 2> "127.0.0.1:10112_solr",
[junit4] 2> "127.0.0.1:10095_solr",
[junit4] 2> "127.0.0.1:10083_solr",
[junit4] 2> "127.0.0.1:10028_solr",
[junit4] 2> "127.0.0.1:10060_solr",
[junit4] 2> "127.0.0.1:10064_solr"]}}
[junit4] 2> 2342634 DEBUG (AutoscalingActionExecutor-9050-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
[junit4] 2> "id":"5c1193e633941Tnu8ynavyq8o6u2intom037rr",
[junit4] 2> "source":"node_lost_trigger",
[junit4] 2> "eventTime":1619689048586561,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 1619689048586561,
[junit4] 2> 1619689044798861,
[junit4] 2> 1619689049846161,
[junit4] 2> 1619689043423561,
[junit4] 2> 1619689046072911,
[junit4] 2> 1619689052384461,
[junit4] 2> 1619689051113211,
[junit4] 2> 1619689047331511],
[junit4] 2> "_enqueue_time_":1619692093440361,
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10043_solr",
[junit4] 2> "127.0.0.1:10061_solr",
[junit4] 2> "127.0.0.1:10112_solr",
[junit4] 2> "127.0.0.1:10095_solr",
[junit4] 2> "127.0.0.1:10083_solr",
[junit4] 2> "127.0.0.1:10028_solr",
[junit4] 2> "127.0.0.1:10060_solr",
[junit4] 2> "127.0.0.1:10064_solr"]}}
[junit4] 2> 2342634 DEBUG (AutoscalingActionExecutor-9050-thread-1) [ ] o.a.s.c.a.ComputePlanAction -- processing event: {
[junit4] 2> "id":"5c1193e633941Tnu8ynavyq8o6u2intom037rr",
[junit4] 2> "source":"node_lost_trigger",
[junit4] 2> "eventTime":1619689048586561,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 1619689048586561,
[junit4] 2> 1619689044798861,
[junit4] 2> 1619689049846161,
[junit4] 2> 1619689043423561,
[junit4] 2> 1619689046072911,
[junit4] 2> 1619689052384461,
[junit4] 2> 1619689051113211,
[junit4] 2> 1619689047331511],
[junit4] 2> "_enqueue_time_":1619692093440361,
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10043_solr",
[junit4] 2> "127.0.0.1:10061_solr",
[junit4] 2> "127.0.0.1:10112_solr",
[junit4] 2> "127.0.0.1:10095_solr",
[junit4] 2> "127.0.0.1:10083_solr",
[junit4] 2> "127.0.0.1:10028_solr",
[junit4] 2> "127.0.0.1:10060_solr",
[junit4] 2> "127.0.0.1:10064_solr"]}} with context properties: {BEFORE_ACTION=[compute]}
[junit4] 2> 2342652 INFO (AutoscalingActionExecutor-9050-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10056_solr&inPlaceMove=true&replica=core_node15
[junit4] 2> 2342664 INFO (AutoscalingActionExecutor-9050-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10072_solr&inPlaceMove=true&replica=core_node28
[junit4] 2> 2342670 INFO (AutoscalingActionExecutor-9050-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10103_solr&inPlaceMove=true&replica=core_node8
[junit4] 2> 2342685 INFO (AutoscalingActionExecutor-9050-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10059_solr&inPlaceMove=true&replica=core_node69
[junit4] 2> 2342688 INFO (AutoscalingActionExecutor-9050-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10053_solr&inPlaceMove=true&replica=core_node70
[junit4] 2> 2342689 INFO (AutoscalingActionExecutor-9050-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10109_solr&inPlaceMove=true&replica=core_node2
[junit4] 2> 2342691 INFO (AutoscalingActionExecutor-9050-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10037_solr&inPlaceMove=true&replica=core_node63
[junit4] 2> 2342692 INFO (AutoscalingActionExecutor-9050-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10100_solr&inPlaceMove=true&replica=core_node52
[junit4] 2> 2342692 INFO (AutoscalingActionExecutor-9050-thread-1) [ ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 1619695024 sessionWrapper.createTime 1619692218952911, this.sessionWrapper.createTime 1619692218952911
[junit4] 2> 2342692 DEBUG (AutoscalingActionExecutor-9050-thread-1) [ ] o.a.s.c.a.ExecutePlanAction -- processing event: {
[junit4] 2> "id":"5c1193e633941Tnu8ynavyq8o6u2intom037rr",
[junit4] 2> "source":"node_lost_trigger",
[junit4] 2> "eventTime":1619689048586561,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 1619689048586561,
[junit4] 2> 1619689044798861,
[junit4] 2> 1619689049846161,
[junit4] 2> 1619689043423561,
[junit4] 2> 1619689046072911,
[junit4] 2> 1619689052384461,
[junit4] 2> 1619689051113211,
[junit4] 2> 1619689047331511],
[junit4] 2> "_enqueue_time_":1619692093440361,
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10043_solr",
[junit4] 2> "127.0.0.1:10061_solr",
[junit4] 2> "127.0.0.1:10112_solr",
[junit4] 2> "127.0.0.1:10095_solr",
[junit4] 2> "127.0.0.1:10083_solr",
[junit4] 2> "127.0.0.1:10028_solr",
[junit4] 2> "127.0.0.1:10060_solr",
[junit4] 2> "127.0.0.1:10064_solr"]}} with context properties: {operations=[org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@119ed07f, org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@1f592c3c, org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@1b66bb0f, org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@5e642607, org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@399e089c, org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@6ff0fb5f, org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@41e08b6d, org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@3e4bd948], AFTER_ACTION=[compute], BEFORE_ACTION=[compute, execute]}
[junit4] 2> 2342692 DEBUG (AutoscalingActionExecutor-9050-thread-1) [ ] o.a.s.c.a.ExecutePlanAction Executing operation: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10056_solr&inPlaceMove=true&replica=core_node15
[junit4] 2> 2342692 DEBUG (AutoscalingActionExecutor-9050-thread-1) [ ] o.a.s.c.a.ExecutePlanAction Saved requestId: node_lost_trigger/5c1193e633941Tnu8ynavyq8o6u2intom037rr/0 in znode: /autoscaling/triggerState/node_lost_trigger/execute/op-0000000000
[junit4] 2> 2342692 DEBUG (simCloudManagerPool-9046-thread-89) [ ] o.a.s.c.a.s.SimCloudManager Invoking Collection Action :movereplica with params ?action=MOVEREPLICA&async=node_lost_trigger%2F5c1193e633941Tnu8ynavyq8o6u2intom037rr%2F0&waitForFinalState=true&collection=testBasic&targetNode=127.0.0.1%3A10056_solr&inPlaceMove=true&replica=core_node15
[junit4] 2> 2342793 DEBUG (simCloudManagerPool-9046-thread-89) [ ] o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node77":{
[junit4] 2> "core":"testBasic_shard1_replica_p76",
[junit4] 2> "shard":"shard1",
[junit4] 2> "collection":"testBasic",
[junit4] 2> "node_name":"127.0.0.1:10056_solr",
[junit4] 2> "type":"PULL"}}
[junit4] 2> 2342903 INFO (simCloudManagerPool-9046-thread-90) [ ] o.a.s.c.ActionThrottle The last leader attempt started 17369ms ago.
[junit4] 2> 2342903 DEBUG (simCloudManagerPool-9046-thread-91) [ ] o.a.s.c.a.s.SimCloudManager Invoking Collection Action :requeststatus with params ?action=REQUESTSTATUS&requestid=node_lost_trigger%2F5c1193e633941Tnu8ynavyq8o6u2intom037rr%2F0
[junit4] 2> 2342903 DEBUG (AutoscalingActionExecutor-9050-thread-1) [ ] o.a.s.c.a.ExecutePlanAction Task with requestId=node_lost_trigger/5c1193e633941Tnu8ynavyq8o6u2intom037rr/0 finished with state=COMPLETED in 0s
[junit4] 2> 2342904 DEBUG (simCloudManagerPool-9046-thread-92) [ ] o.a.s.c.a.s.SimCloudManager Invoking Collection Action :deletestatus with params ?action=DELETESTATUS&requestid=node_lost_trigger%2F5c1193e633941Tnu8ynavyq8o6u2intom037rr%2F0
[junit4] 2> 2342904 DEBUG (AutoscalingActionExecutor-9050-thread-1) [ ] o.a.s.c.a.ExecutePlanAction Executing operation: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10072_solr&inPlaceMove=true&replica=core_node28
[junit4] 2> 2342904 DEBUG (AutoscalingActionExecutor-9050-thread-1) [ ] o.a.s.c.a.ExecutePlanAction Saved requestId: node_lost_trigger/5c1193e633941Tnu8ynavyq8o6u2intom037rr/0 in znode: /autoscaling/triggerState/node_lost_trigger/execute/op-0000000001
[junit4] 2> 2342904 DEBUG (simCloudManagerPool-9046-thread-93) [ ] o.a.s.c.a.s.SimCloudManager Invoking Collection Action :movereplica with params ?action=MOVEREPLICA&async=node_lost_trigger%2F5c1193e633941Tnu8ynavyq8o6u2intom037rr%2F0&waitForFinalState=true&collection=testBasic&targetNode=127.0.0.1%3A10072_solr&inPlaceMove=true&replica=core_node28
[junit4] 2> 2343005 DEBUG (simCloudManagerPool-9046-thread-93) [ ] o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node79":{
[junit4] 2> "core":"testBasic_shard2_replica_p78",
[junit4] 2> "shard":"shard2",
[junit4] 2> "collection":"testBasic",
[junit4] 2> "node_name":"127.0.0.1:10072_solr",
[junit4] 2> "type":"PULL"}}
[junit4] 2> 2343115 INFO (simCloudManagerPool-9046-thread-94) [ ] o.a.s.c.ActionThrottle The last leader attempt started 10595ms ago.
[junit4] 2> 2343115 DEBUG (simCloudManagerPool-9046-thread-95) [ ] o.a.s.c.a.s.SimCloudManager Invoking Collection Action :requeststatus with params ?action=REQUESTSTATUS&requestid=node_lost_trigger%2F5c1193e633941Tnu8ynavyq8o6u2intom037rr%2F0
[junit4] 2> 2343116 DEBUG (AutoscalingActionExecutor-9050-thread-1) [ ] o.a.s.c.a.ExecutePlanAction Task with requestId=node_lost_trigger/5c1193e633941Tnu8ynavyq8o6u2intom037rr/0 finished with state=COMPLETED in 0s
[junit4] 2> 2343116 DEBUG (simCloudManagerPool-9046-thread-96) [ ] o.a.s.c.a.s.SimCloudManager Invoking Collection Action :deletestatus with params ?action=DELETESTATUS&requestid=node_lost_trigger%2F5c1193e633941Tnu8ynavyq8o6u2intom037rr%2F0
[junit4] 2> 2343116 DEBUG (AutoscalingActionExecutor-9050-thread-1) [ ] o.a.s.c.a.ExecutePlanAction Executing operation: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10103_solr&inPlaceMove=true&replica=core_node8
[junit4] 2> 2343117 DEBUG (AutoscalingActionExecutor-9050-thread-1) [ ] o.a.s.c.a.ExecutePlanAction Saved requestId: node_lost_trigger/5c1193e633941Tnu8ynavyq8o6u2intom037rr/0 in znode: /autoscaling/triggerState/node_lost_trigger/execute/op-0000000002
[junit4] 2> 2343117 DEBUG (simCloudManagerPool-9046-thread-97) [ ] o.a.s.c.a.s.SimCloudManager Invoking Collection Action :movereplica with params ?action=MOVEREPLICA&async=node_lost_trigger%2F5c1193e633941Tnu8ynavyq8o6u2intom037rr%2F0&waitForFinalState=true&collection=testBasic&targetNode=127.0.0.1%3A10103_solr&inPlaceMove=true&replica=core_node8
[junit4] 2> 2343218 DEBUG (simCloudManagerPool-9046-thread-97) [ ] o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node81":{
[junit4] 2> "core":"testBasic_shard1_replica_t80",
[junit4] 2> "shard":"shard1",
[junit4] 2> "collection":"testBasic",
[junit4] 2> "node_name":"127.0.0.1:10103_solr",
[junit4] 2> "type":"TLOG"}}
[junit4] 2> 2343329 INFO (simCloudManagerPool-9046-thread-98) [ ] o.a.s.c.ActionThrottle The last leader attempt started 10664ms ago.
[junit4] 2> 2343329 DEBUG (simCloudManagerPool-9046-thread-99) [ ] o.a.s.c.a.s.SimCloudManager Invoking Collection Action :requeststatus with params ?action=REQUESTSTATUS&requestid=node_lost_trigger%2F5c1193e633941Tnu8ynavyq8o6u2intom037rr%2F0
[junit4] 2> 2343329 DEBUG (AutoscalingActionExecutor-9050-thread-1) [ ] o.a.s.c.a.ExecutePlanAction Task with requestId=node_lost_trigger/5c1193e633941Tnu8ynavyq8o6u2intom037rr/0 finished with state=COMPLETED in 0s
[junit4] 2> 2343329 DEBUG (simCloudManagerPool-9046-thread-100) [ ] o.a.s.c.a.s.SimCloudManager Invoking Collection Action :deletestatus with params ?action=DELETESTATUS&requestid=node_lost_trigger%2F5c1193e633941Tnu8ynavyq8o6u2intom037rr%2F0
[junit4] 2> 2343329 DEBUG (AutoscalingActionExecutor-9050-thread-1) [ ] o.a.s.c.a.ExecutePlanAction Executing operation: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10059_solr&inPlaceMove=true&replica=core_node69
[junit4] 2> 2343330 DEBUG (AutoscalingActionExecutor-9050-thread-1) [ ] o.a.s.c.a.ExecutePlanAction Saved requestId: node_lost_trigger/5c1193e633941Tnu8ynavyq8o6u2intom037rr/0 in znode: /autoscaling/triggerState/node_lost_trigger/execute/op-0000000003
[junit4] 2> 2343330 DEBUG (simCloudManagerPool-9046-thread-101) [ ] o.a.s.c.a.s.SimCloudManager Invoking Collection Action :movereplica with params ?action=MOVEREPLICA&async=node_lost_trigger%2F5c1193e633941Tnu8ynavyq8o6u2intom037rr%2F0&waitForFinalState=true&collection=testBasic&targetNode=127.0.0.1%3A10059_solr&inPlaceMove=true&replica=core_node69
[junit4] 2> 2343431 DEBUG (simCloudManagerPool-9046-thread-101) [ ] o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node83":{
[junit4] 2> "core":"testBasic_shard5_replica_t82",
[junit4] 2> "shard":"shard5",
[junit4] 2> "collection":"testBasic",
[junit4] 2> "node_name":"127.0.0.1:10059_solr",
[junit4] 2> "type":"TLOG"}}
[junit4] 2> 2343541 INFO (simCloudManagerPool-9046-thread-102) [ ] o.a.s.c.ActionThrottle The last leader attempt started 10632ms ago.
[junit4] 2> 2343541 DEBUG (simCloudManagerPool-9046-thread-103) [ ] o.a.s.c.a.s.SimCloudManager Invoking Collection Action :requeststatus with params ?action=REQUESTSTATUS&requestid=node_lost_trigger%2F5c1193e633941Tnu8ynavyq8o6u2intom037rr%2F0
[junit4] 2> 2343541 DEBUG (AutoscalingActionExecutor-9050-thread-1) [ ] o.a.s.c.a.ExecutePlanAction Task with requestId=node_lost_trigger/5c1193e633941Tnu8ynavyq8o6u2intom037rr/0 finished with state=COMPLETED in 0s
[junit4] 2> 2343542 DEBUG (simCloudManagerPool-9046-thread-104) [ ] o.a.s.c.a.s.SimCloudManager Invoking Collection Action :deletestatus with params ?action=DELETESTATUS&requestid=node_lost_trigger%2F5c1193e633941Tnu8ynavyq8o6u2intom037rr%2F0
[junit4] 2> 2343542 DEBUG (AutoscalingActionExecutor-9050-thread-1) [ ] o.a.s.c.a.ExecutePlanAction Executing operation: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10053_solr&inPlaceMove=true&replica=core_node70
[junit4] 2> 2343542 DEBUG (AutoscalingActionExecutor-9050-thread-1) [ ] o.a.s.c.a.ExecutePlanAction Saved requestId: node_lost_trigger/5c1193e633941Tnu8ynavyq8o6u2intom037rr/0 in znode: /autoscaling/triggerState/node_lost_trigger/execute/op-0000000004
[junit4] 2> 2343542 DEBUG (simCloudManagerPool-9046-thread-105) [ ] o.a.s.c.a.s.SimCloudManager Invoking Collection Action :movereplica with params ?action=MOVEREPLICA&async=node_lost_trigger%2F5c1193e633941Tnu8ynavyq8o6u2intom037rr%2F0&waitForFinalState=true&collection=testBasic&targetNode=127.0.0.1%3A10053_solr&inPlaceMove=true&replica=core_node70
[junit4] 2> 2343642 DEBUG (simCloudManagerPool-9046-thread-105) [ ] o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node85":{
[junit4] 2> "core":"testBasic_shard5_replica_t84",
[junit4] 2> "shard":"shard5",
[junit4] 2> "collection":"testBasic",
[junit4] 2> "node_name":"127.0.0.1:10053_solr",
[junit4] 2> "type":"TLOG"}}
[junit4] 2> 2343753 INFO (simCloudManagerPool-9046-thread-106) [ ] o.a.s.c.ActionThrottle The last leader attempt started 10576ms ago.
[junit4] 2> 2343753 DEBUG (simCloudManagerPool-9046-thread-107) [ ] o.a.s.c.a.s.SimCloudManager Invoking Collection Action :requeststatus with params ?action=REQUESTSTATUS&requestid=node_lost_trigger%2F5c1193e633941Tnu8ynavyq8o6u2intom037rr%2F0
[junit4] 2> 2343753 DEBUG (AutoscalingActionExecutor-9050-thread-1) [ ] o.a.s.c.a.ExecutePlanAction Task with requestId=node_lost_trigger/5c1193e633941Tnu8ynavyq8o6u2intom037rr/0 finished with state=COMPLETED in 0s
[junit4] 2> 2343754 DEBUG (simCloudManagerPool-9046-thread-108) [ ] o.a.s.c.a.s.SimCloudManager Invoking Collection Action :deletestatus with params ?action=DELETESTATUS&requestid=node_lost_trigger%2F5c1193e633941Tnu8ynavyq8o6u2intom037rr%2F0
[junit4] 2> 2343754 DEBUG (AutoscalingActionExecutor-9050-thread-1) [ ] o.a.s.c.a.ExecutePlanAction Executing operation: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10109_solr&inPlaceMove=true&replica=core_node2
[junit4] 2> 2343754 DEBUG (AutoscalingActionExecutor-9050-thread-1) [ ] o.a.s.c.a.ExecutePlanAction Saved requestId: node_lost_trigger/5c1193e633941Tnu8ynavyq8o6u2intom037rr/0 in znode: /autoscaling/triggerState/node_lost_trigger/execute/op-0000000005
[junit4] 2> 2343755 DEBUG (simCloudManagerPool-9046-thread-109) [ ] o.a.s.c.a.s.SimCloudManager Invoking Collection Action :movereplica with params ?action=MOVEREPLICA&async=node_lost_trigger%2F5c1193e633941Tnu8ynavyq8o6u2intom037rr%2F0&waitForFinalState=true&collection=testBasic&targetNode=127.0.0.1%3A10109_solr&inPlaceMove=true&replica=core_node2
[junit4] 2> 2343856 DEBUG (simCloudManagerPool-9046-thread-109) [ ] o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node87":{
[junit4] 2> "core":"testBasic_shard1_replica_n86",
[junit4] 2> "shard":"shard1",
[junit4] 2> "collection":"testBasic",
[junit4] 2> "node_name":"127.0.0.1:10109_solr",
[junit4] 2> "type":"NRT"}}
[junit4] 2> 2343966 INFO (simCloudManagerPool-9046-thread-110) [ ] o.a.s.c.ActionThrottle The last leader attempt started 10664ms ago.
[junit4] 2> 2343967 DEBUG (simCloudManagerPool-9046-thread-111) [ ] o.a.s.c.a.s.SimCloudManager Invoking Collection Action :requeststatus with params ?action=REQUESTSTATUS&requestid=node_lost_trigger%2F5c1193e633941Tnu8ynavyq8o6u2intom037rr%2F0
[junit4] 2> 2343967 DEBUG (AutoscalingActionExecutor-9050-thread-1) [ ] o.a.s.c.a.ExecutePlanAction Task with requestId=node_lost_trigger/5c1193e633941Tnu8ynavyq8o6u2intom037rr/0 finished with state=COMPLETED in 0s
[junit4] 2> 2343967 DEBUG (simCloudManagerPool-9046-thread-112) [ ] o.a.s.c.a.s.SimCloudManager Invoking Collection Action :deletestatus with params ?action=DELETESTATUS&requestid=node_lost_trigger%2F5c1193e633941Tnu8ynavyq8o6u2intom037rr%2F0
[junit4] 2> 2343967 DEBUG (AutoscalingActionExecutor-9050-thread-1) [ ] o.a.s.c.a.ExecutePlanAction Executing operation: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10037_solr&inPlaceMove=true&replica=core_node63
[junit4] 2> 2343967 DEBUG (AutoscalingActionExecutor-9050-thread-1) [ ] o.a.s.c.a.ExecutePlanAction Saved requestId: node_lost_trigger/5c1193e633941Tnu8ynavyq8o6u2intom037rr/0 in znode: /autoscaling/triggerState/node_lost_trigger/execute/op-0000000006
[junit4] 2> 2343968 DEBUG (simCloudManagerPool-9046-thread-113) [ ] o.a.s.c.a.s.SimCloudManager Invoking Collection Action :movereplica with params ?action=MOVEREPLICA&async=node_lost_trigger%2F5c1193e633941Tnu8ynavyq8o6u2intom037rr%2F0&waitForFinalState=true&collection=testBasic&targetNode=127.0.0.1%3A10037_solr&inPlaceMove=true&replica=core_node63
[junit4] 2> 2343971 WARN (AutoscalingActionExecutor-9050-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Exception executing actions
[junit4] 2> java.lang.Exception: Error executing action: execute for trigger event: {
[junit4] 2> "id":"5c1193e633941Tnu8ynavyq8o6u2intom037rr",
[junit4] 2> "source":"node_lost_trigger",
[junit4] 2> "eventTime":1619689048586561,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 1619689048586561,
[junit4] 2> 1619689044798861,
[junit4] 2> 1619689049846161,
[junit4] 2> 1619689043423561,
[junit4] 2> 1619689046072911,
[junit4] 2> 1619689052384461,
[junit4] 2> 1619689051113211,
[junit4] 2> 1619689047331511],
[junit4] 2> "_enqueue_time_":1619692093440361,
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10043_solr",
[junit4] 2> "127.0.0.1:10061_solr",
[junit4] 2> "127.0.0.1:10112_solr",
[junit4] 2> "127.0.0.1:10095_solr",
[junit4] 2> "127.0.0.1:10083_solr",
[junit4] 2> "127.0.0.1:10028_solr",
[junit4] 2> "127.0.0.1:10060_solr",
[junit4] 2> "127.0.0.1:10064_solr"]}}
[junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$add$3(ScheduledTriggers.java:330)
[junit4] 2> at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:514)
[junit4] 2> at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
[junit4] 2> at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1135)
[junit4] 2> at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
[junit4] 2> at java.base/java.lang.Thread.run(Thread.java:844)
[junit4] 2> Caused by: org.apache.solr.common.SolrException: Unexpected exception while processing event: {
[junit4] 2> "id":"5c1193e633941Tnu8ynavyq8o6u2intom037rr",
[junit4] 2> "source":"node_lost_trigger",
[junit4] 2> "eventTime":1619689048586561,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 1619689048586561,
[junit4] 2> 1619689044798861,
[junit4] 2> 1619689049846161,
[junit4] 2> 1619689043423561,
[junit4] 2> 1619689046072911,
[junit4] 2> 1619689052384461,
[junit4] 2> 1619689051113211,
[junit4] 2> 1619689047331511],
[junit4] 2> "_enqueue_time_":1619692093440361,
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10043_solr",
[junit4] 2> "127.0.0.1:10061_solr",
[junit4] 2> "127.0.0.1:10112_solr",
[junit4] 2> "127.0.0.1:10095_solr",
[junit4] 2> "127.0.0.1:10083_solr",
[junit4] 2> "127.0.0.1:10028_solr",
[junit4] 2> "127.0.0.1:10060_solr",
[junit4] 2> "127.0.0.1:10064_solr"]}}
[junit4] 2> at org.apache.solr.cloud.autoscaling.ExecutePlanAction.process(ExecutePlanAction.java:120)
[junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$add$3(ScheduledTriggers.java:327)
[junit4] 2> ... 6 more
[junit4] 2> Caused by: org.apache.solr.common.SolrException: Unexpected exception executing operation: action=MOVEREPLICA&async=node_lost_trigger/5c1193e633941Tnu8ynavyq8o6u2intom037rr/0&waitForFinalState=true&collection=testBasic&targetNode=127.0.0.1:10037_solr&inPlaceMove=true&replica=core_node63
[junit4] 2> at org.apache.solr.cloud.autoscaling.ExecutePlanAction.process(ExecutePlanAction.java:110)
[junit4] 2> ... 7 more
[junit4] 2> Caused by: java.io.IOException: java.util.concurrent.ExecutionException: org.apache.solr.common.SolrException: java.io.IOException: org.apache.solr.client.solrj.cloud.autoscaling.BadVersionException: /clusterstate.json
[junit4] 2> at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.request(SimCloudManager.java:430)
[junit4] 2> at org.apache.solr.cloud.autoscaling.ExecutePlanAction.process(ExecutePlanAction.java:80)
[junit4] 2> ... 7 more
[junit4] 2> Caused by: java.util.concurrent.ExecutionException: org.apache.solr.common.SolrException: java.io.IOException: org.apache.solr.client.solrj.cloud.autoscaling.BadVersionException: /clusterstate.json
[junit4] 2> at java.base/java.util.concurrent.FutureTask.report(FutureTask.java:122)
[junit4] 2> at java.base/java.util.concurrent.FutureTask.get(FutureTask.java:191)
[junit4] 2> at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.request(SimCloudManager.java:428)
[junit4] 2> ... 8 more
[junit4] 2> Caused by: org.apache.solr.common.SolrException: java.io.IOException: org.apache.solr.client.solrj.cloud.autoscaling.BadVersionException: /clusterstate.json
[junit4] 2> at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.simHandleSolrRequest(SimCloudManager.java:547)
[junit4] 2> at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.lambda$request$0(SimCloudManager.java:427)
[junit4] 2> ... 5 more
[junit4] 2> Caused by: java.io.IOException: org.apache.solr.client.solrj.cloud.autoscaling.BadVersionException: /clusterstate.json
[junit4] 2> at org.apache.solr.cloud.autoscaling.sim.SimClusterStateProvider.saveClusterState(SimClusterStateProvider.java:494)
[junit4] 2> at org.apache.solr.cloud.autoscaling.sim.SimClusterStateProvider.getClusterState(SimClusterStateProvider.java:1192)
[junit4] 2> at org.apache.solr.cloud.autoscaling.sim.SimClusterStateProvider.simMoveReplica(SimClusterStateProvider.java:739)
[junit4] 2> at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.simHandleSolrRequest(SimCloudManager.java:545)
[junit4] 2> ... 6 more
[junit4] 2> Caused by: org.apache.solr.client.solrj.cloud.autoscaling.BadVersionException: /clusterstate.json
[junit4] 2> at org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node.setData(SimDistribStateManager.java:114)
[junit4] 2> at org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager.setData(SimDistribStateManager.java:502)
[junit4] 2> at org.apache.solr.cloud.autoscaling.sim.SimClusterStateProvider.saveClusterState(SimClusterStateProvider.java:490)
[junit4] 2> ... 9 more
[junit4] 2> 2343972 DEBUG (AutoscalingActionExecutor-9050-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 66910 ms for event id=5c1193e633941Tnu8ynavyq8o6u2intom037rr
[junit4] 2> 2343975 DEBUG (ScheduledTrigger-9049-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 92
[junit4] 2> 2343976 DEBUG (ScheduledTrigger-9049-thread-2) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:10043_solr, 127.0.0.1:10061_solr, 127.0.0.1:10112_solr, 127.0.0.1:10095_solr, 127.0.0.1:10083_solr, 127.0.0.1:10028_solr, 127.0.0.1:10060_solr, 127.0.0.1:10064_solr]
[junit4] 2> 2343976 DEBUG (ScheduledTrigger-9049-thread-2) [ ] o.a.s.c.a.ScheduledTriggers -------- Cooldown period - rejecting event: {
[junit4] 2> "id":"5c11934191055Tnu8ynavyq8o6u2intom037ru",
[junit4] 2> "source":".auto_add_replicas",
[junit4] 2> "eventTime":1619688875954261,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 1619688875954261,
[junit4] 2> 1619688871479861,
[junit4] 2> 1619688877234661,
[junit4] 2> 1619688870053611,
[junit4] 2> 1619688873058661,
[junit4] 2> 1619688879784761,
[junit4] 2> 1619688878508561,
[junit4] 2> 1619688874628461],
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10043_solr",
[junit4] 2> "127.0.0.1:10061_solr",
[junit4] 2> "127.0.0.1:10112_solr",
[junit4] 2> "127.0.0.1:10095_solr",
[junit4] 2> "127.0.0.1:10083_solr",
[junit4] 2> "127.0.0.1:10028_solr",
[junit4] 2> "127.0.0.1:10060_solr",
[junit4] 2> "127.0.0.1:10064_solr"]}}
[junit4] 2> 2343976 DEBUG (ScheduledTrigger-9049-thread-2) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger listener for lost nodes: [127.0.0.1:10043_solr, 127.0.0.1:10061_solr, 127.0.0.1:10112_solr, 127.0.0.1:10095_solr, 127.0.0.1:10083_solr, 127.0.0.1:10028_solr, 127.0.0.1:10060_solr, 127.0.0.1:10064_solr] is not ready, will try later
[junit4] 2> 2343987 DEBUG (ScheduledTrigger-9049-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 92
[junit4] 2> 2343997 DEBUG (ScheduledTrigger-9049-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 92
[junit4] 2> 2343997 DEBUG (ScheduledTrigger-9049-thread-1) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:10043_solr, 127.0.0.1:10061_solr, 127.0.0.1:10112_solr, 127.0.0.1:10095_solr, 127.0.0.1:10083_solr, 127.0.0.1:10028_solr, 127.0.0.1:10060_solr, 127.0.0.1:10064_solr]
[junit4] 2> 2343997 DEBUG (ScheduledTrigger-9049-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -------- Cooldown period - rejecting event: {
[junit4] 2> "id":"5c11934191055Tnu8ynavyq8o6u2intom037rw",
[junit4] 2> "source":".auto_add_replicas",
[junit4] 2> "eventTime":1619688875954261,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 1619688875954261,
[junit4] 2> 1619688871479861,
[junit4] 2> 1619688877234661,
[junit4] 2> 1619688870053611,
[junit4] 2> 1619688873058661,
[junit4] 2> 1619688879784761,
[junit4] 2> 1619688878508561,
[junit4] 2> 1619688874628461],
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10043_solr",
[junit4] 2> "127.0.0.1:10061_solr",
[junit4] 2> "127.0.0.1:10112_solr",
[junit4] 2> "127.0.0.1:10095_solr",
[junit4] 2> "127.0.0.1:10083_solr",
[junit4] 2> "127.0.0.1:10028_solr",
[junit4] 2> "127.0.0.1:10060_solr",
[junit4] 2> "127.0.0.1:10064_solr"]}}
[junit4] 2> 2343997 DEBUG (ScheduledTrigger-9049-thread-1) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger listener for lost nodes: [127.0.0.1:10043_solr, 127.0.0.1:10061_solr, 127.0.0.1:10112_solr, 127.0.0.1:10095_solr, 127.0.0.1:10083_solr, 127.0.0.1:10028_solr, 127.0.0.1:10060_solr, 127.0.0.1:10064_solr] is not ready, will try later
[junit4] 2> 2344007 DEBUG (ScheduledTrigger-9049-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 92
[junit4] 2> 2344018 DEBUG (ScheduledTrigger-9049-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 92
[junit4] 2> 2344018 DEBUG (ScheduledTrigger-9049-thread-2) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:10043_solr, 127.0.0.1:10061_solr, 127.0.0.1:10112_solr, 127.0.0.1:10095_solr, 127.0.0.1:10083_solr, 127.0.0.1:10028_solr, 127.0.0.1:10060_solr, 127.0.0.1:10064_solr]
[junit4] 2> 2344018 DEBUG (ScheduledTrigger-9049-thread-2) [ ] o.a.s.c.a.ScheduledTriggers -------- Cooldown period - rejecting event: {
[junit4] 2> "id":"5c11934191055Tnu8ynavyq8o6u2intom037ry",
[junit4] 2> "source":".auto_add_replicas",
[junit4] 2> "eventTime":1619688875954261,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 1619688875954261,
[junit4] 2> 1619688871479861,
[junit4] 2> 1619688877234661,
[junit4] 2> 1619688870053611,
[junit4] 2> 1619688873058661,
[junit4] 2> 1619688879784761,
[junit4] 2> 1619688878508561,
[junit4] 2> 1619688874628461],
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10043_solr",
[junit4] 2> "127.0.0.1:10061_solr",
[junit4] 2> "127.0.0.1:10112_solr",
[junit4] 2> "127.0.0.1:10095_solr",
[junit4] 2> "127.0.0.1:10083_solr",
[junit4] 2> "127.0.0.1:10028_solr",
[junit4] 2> "127.0.0.1:10060_solr",
[junit4] 2> "127.0.0.1:10064_solr"]}}
[junit4] 2> 2344018 DEBUG (ScheduledTrigger-9049-thread-2) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger listener for lost nodes: [127.0.0.1:10043_solr, 127.0.0.1:10061_solr, 127.0.0.1:10112_solr, 127.0.0.1:10095_solr, 127.0.0.1:10083_solr, 127.0.0.1:10028_solr, 127.0.0.1:10060_solr, 127.0.0.1:10064_solr] is not ready, will try later
[junit4] 2> 2344028 DEBUG (ScheduledTrigger-9049-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 92
[junit4] 2> 2344039 DEBUG (ScheduledTrigger-9049-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 92
[junit4] 2> 2344039 DEBUG (ScheduledTrigger-9049-thread-1) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:10043_solr, 127.0.0.1:10061_solr, 127.0.0.1:10112_solr, 127.0.0.1:10095_solr, 127.0.0.1:10083_solr, 127.0.0.1:10028_solr, 127.0.0.1:10060_solr, 127.0.0.1:10064_solr]
[junit4] 2> 2344039 DEBUG (ScheduledTrigger-9049-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -------- Cooldown period - rejecting event: {
[junit4] 2> "id":"5c11934191055Tnu8ynavyq8o6u2intom037s0",
[junit4] 2> "source":".auto_add_replicas",
[junit4] 2> "eventTime":1619688875954261,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 1619688875954261,
[junit4] 2> 1619688871479861,
[junit4] 2> 1619688877234661,
[junit4] 2> 1619688870053611,
[junit4] 2> 1619688873058661,
[junit4] 2> 1619688879784761,
[junit4] 2> 1619688878508561,
[junit4] 2> 1619688874628461],
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10043_solr",
[junit4] 2> "127.0.0.1:10061_solr",
[junit4] 2> "127.0.0.1:10112_solr",
[junit4] 2>
[...truncated too long message...]
y, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@72964586
[junit4] 2> 2690824 INFO (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[472B651A27B2B5E8]) [ ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:44235_r
[junit4] 2> 2690824 INFO (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[472B651A27B2B5E8]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
[junit4] 2> 2690824 INFO (zkCallback-5722-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
[junit4] 2> 2690825 INFO (zkCallback-5683-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
[junit4] 2> 2690825 INFO (coreCloseExecutor-7333-thread-1) [ ] o.a.s.c.SolrCore [collection1_shard1_replica_t47] CLOSING SolrCore org.apache.solr.core.SolrCore@967ff32
[junit4] 2> 2690825 INFO (coreCloseExecutor-7333-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1.shard1.replica_t47, tag=157810482
[junit4] 2> 2690825 INFO (coreCloseExecutor-7333-thread-1) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@576e7c47: rootName = null, domain = solr.core.collection1.shard1.replica_t47, service url = null, agent id = null] for registry solr.core.collection1.shard1.replica_t47 / com.codahale.metrics.MetricRegistry@55b3792d
[junit4] 2> 2690825 INFO (zkCallback-5722-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [0])
[junit4] 2> 2690831 INFO (coreCloseExecutor-7333-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.collection1.shard1.leader, tag=157810482
[junit4] 2> 2690832 INFO (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[472B651A27B2B5E8]) [ ] o.a.s.c.Overseer Overseer (id=99209247056527378-127.0.0.1:44235_r-n_0000000004) closing
[junit4] 2> 2690832 INFO (OverseerStateUpdate-99209247056527378-127.0.0.1:44235_r-n_0000000004) [ ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:44235_r
[junit4] 2> 2690832 WARN (OverseerAutoScalingTriggerThread-99209247056527378-127.0.0.1:44235_r-n_0000000004) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 2690834 INFO (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[472B651A27B2B5E8]) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@610d96d9{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 2690834 INFO (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[472B651A27B2B5E8]) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@2db95853{/r,null,UNAVAILABLE}
[junit4] 2> 2690835 INFO (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[472B651A27B2B5E8]) [ ] o.a.s.c.ChaosMonkey monkey: stop jetty! 34999
[junit4] 2> 2690835 ERROR (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[472B651A27B2B5E8]) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 2690836 INFO (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[472B651A27B2B5E8]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:37863 37863
[junit4] 2> 2705860 INFO (Thread-5017) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:37863 37863
[junit4] 2> 2705861 WARN (Thread-5017) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 8 /solr/aliases.json
[junit4] 2> 6 /solr/security.json
[junit4] 2> 6 /solr/configs/conf1
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 8 /solr/clusterprops.json
[junit4] 2> 8 /solr/clusterstate.json
[junit4] 2> 7 /solr/collections/collection1/state.json
[junit4] 2> 4 /solr/autoscaling.json
[junit4] 2> 2 /solr/overseer_elect/election/99209247056527369-127.0.0.1:35655_r-n_0000000001
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 8 /solr/live_nodes
[junit4] 2> 8 /solr/collections
[junit4] 2> 4 /solr/overseer/queue
[junit4] 2> 4 /solr/autoscaling/events/.auto_add_replicas
[junit4] 2> 4 /solr/overseer/queue-work
[junit4] 2> 4 /solr/overseer/collection-queue-work
[junit4] 2>
[junit4] 2> NOTE: leaving temporary files on disk at: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.ShardSplitTest_472B651A27B2B5E8-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70), sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@466c4290), locale=sv-AX, timezone=PST8PDT
[junit4] 2> NOTE: Linux 4.10.0-40-generic amd64/Oracle Corporation 10-ea (64-bit)/cpus=8,threads=1,free=60459624,total=518979584
[junit4] 2> NOTE: All tests run in this JVM: [TestLeaderElectionZkExpiry, CoreAdminRequestStatusTest, FieldMutatingUpdateProcessorTest, RecoveryZkTest, TestFieldSortValues, TestGenericDistributedQueue, SpellCheckCollatorWithCollapseTest, TestDistributedGrouping, SortByFunctionTest, TestEmbeddedSolrServerConstructors, EchoParamsTest, CoreSorterTest, TestSolrXml, HdfsWriteToMultipleCollectionsTest, TestStressInPlaceUpdates, ClusterStateTest, RollingRestartTest, SolrIndexMetricsTest, BlockCacheTest, TestFileDictionaryLookup, IndexSchemaTest, NodeLostTriggerTest, TestShardHandlerFactory, DistributedFacetPivotSmallTest, TestFieldCacheSortRandom, TestDistributedMissingSort, TestDynamicFieldCollectionResource, TestSizeLimitedDistributedMap, TestPivotHelperCode, BadComponentTest, TestSolrCLIRunExample, CurrencyFieldTypeTest, FullSolrCloudDistribCmdsTest, EnumFieldTest, TestSolrConfigHandlerConcurrent, TestQuerySenderListener, MoveReplicaHDFSFailoverTest, TestSolrFieldCacheBean, HdfsThreadLeakTest, AnalyticsMergeStrategyTest, TestClassicSimilarityFactory, TestSchemaNameResource, HdfsDirectoryTest, TestUninvertingReader, TestSQLHandler, PluginInfoTest, TestExtendedDismaxParser, TestPolicyCloud, TestPrepRecovery, StatsComponentTest, TestTriggerIntegration, TestMaxScoreQueryParser, ClassificationUpdateProcessorIntegrationTest, DistributedDebugComponentTest, BitVectorTest, BlockJoinFacetSimpleTest, CSVRequestHandlerTest, DocValuesNotIndexedTest, TestElisionMultitermQuery, TestConfigReload, TestCharFilters, TestCoreContainer, TestReplicaProperties, SegmentsInfoRequestHandlerTest, CopyFieldTest, TestFieldCache, CdcrBootstrapTest, TestLegacyField, PKIAuthenticationIntegrationTest, TestZkChroot, TestHalfAndHalfDocValues, TestNodeLostTrigger, TestLegacyFieldCache, SystemInfoHandlerTest, TestSearchPerf, TestMinMaxOnMultiValuedField, CollectionReloadTest, TestNestedDocsSort, TestMacroExpander, SolrCLIZkUtilsTest, DeleteStatusTest, ZkNodePropsTest, DistanceFunctionTest, TestExportWriter, TestCollectionAPI, DirectSolrConnectionTest, TestLRUCache, TestCloudManagedSchema, ChaosMonkeyNothingIsSafeTest, TestFieldResource, SuggesterWFSTTest, TestCollectionAPIs, TestTrie, ZkFailoverTest, TestSSLRandomization, TestExclusionRuleCollectionAccess, TestDeleteCollectionOnDownNodes, TestPerFieldSimilarityWithDefaultOverride, TestHdfsUpdateLog, TestDistributedStatsComponentCardinality, SortSpecParsingTest, BadCopyFieldTest, WordBreakSolrSpellCheckerTest, TestFuzzyAnalyzedSuggestions, TestLMDirichletSimilarityFactory, TestLegacyTerms, TestComplexPhraseLeadingWildcard, TestConfigSets, FileBasedSpellCheckerTest, UnloadDistributedZkTest, LeaderFailureAfterFreshStartTest, UniqFieldsUpdateProcessorFactoryTest, QueryElevationComponentTest, DistributedQueryComponentOptimizationTest, TestSchemalessBufferedUpdates, OverriddenZkACLAndCredentialsProvidersTest, SuggesterTest, LargeFieldTest, TestCloudJSONFacetJoinDomain, SolrCoreCheckLockOnStartupTest, TestConfigSetsAPIExclusivity, TestReload, TestHashQParserPlugin, LeaderElectionIntegrationTest, TestRandomFaceting, ZkSolrClientTest, TestRealTimeGet, TestJoin, TestSolr4Spatial, SpellCheckComponentTest, TestSort, TestFunctionQuery, DirectUpdateHandlerTest, TestIndexSearcher, HighlighterTest, ShowFileRequestHandlerTest, DistributedQueryElevationComponentTest, SolrIndexSplitterTest, SimplePostToolTest, SpellCheckCollatorTest, CoreAdminHandlerTest, SuggesterTSTTest, SpatialFilterTest, PolyFieldTest, TestPseudoReturnFields, TestUpdate, QueryEqualityTest, DirectUpdateHandlerOptimizeTest, DefaultValueUpdateProcessorTest, SolrInfoBeanTest, CacheHeaderTest, TestSurroundQueryParser, LukeRequestHandlerTest, DirectSolrSpellCheckerTest, TestQueryTypes, XmlUpdateRequestHandlerTest, RequestHandlersTest, DocumentBuilderTest, TestValueSourceCache, RequiredFieldsTest, TestSolrQueryParser, LoggingHandlerTest, ReturnFieldsTest, TestCSVResponseWriter, TestAnalyzedSuggestions, BinaryUpdateRequestHandlerTest, TestLFUCache, UUIDFieldTest, DistributedMLTComponentTest, SolrTestCaseJ4Test, TestCrossCoreJoin, TestCursorMarkWithoutUniqueKey, TestTolerantSearch, TestLuceneIndexBackCompat, ActionThrottleTest, AliasIntegrationTest, ChaosMonkeySafeLeaderWithPullReplicasTest, CloudExitableDirectoryReaderTest, CollectionTooManyReplicasTest, ConcurrentDeleteAndCreateCollectionTest, ConfigSetsAPITest, ConnectionManagerTest, DeleteInactiveReplicaTest, DeleteNodeTest, DistributedVersionInfoTest, LeaderInitiatedRecoveryOnCommitTest, MultiThreadedOCPTest, ReplaceNodeTest, ReplicationFactorTest, RestartWhileUpdatingTest, SSLMigrationTest, ShardSplitTest]
[junit4] Completed [684/764 (2!)] on J2 in 768.88s, 5 tests, 1 failure <<< FAILURES!
[...truncated 41047 lines...]