You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2018/01/23 20:30:17 UTC
[JENKINS-EA] Lucene-Solr-7.x-Linux (64bit/jdk-10-ea+37) - Build #
1229 - Still Unstable!
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Linux/1229/
Java: 64bit/jdk-10-ea+37 -XX:+UseCompressedOops -XX:+UseParallelGC
2 tests failed.
FAILED: org.apache.solr.client.solrj.io.stream.StreamExpressionTest.testFuzzyKmeans
Error Message:
expected:<1.3724537218127956E-4> but was:<0.0>
Stack Trace:
java.lang.AssertionError: expected:<1.3724537218127956E-4> but was:<0.0>
at __randomizedtesting.SeedInfo.seed([6471768B25648F68:FEA595A4E052219]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:443)
at org.junit.Assert.assertEquals(Assert.java:512)
at org.apache.solr.client.solrj.io.stream.StreamExpressionTest.testFuzzyKmeans(StreamExpressionTest.java:7140)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:564)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.base/java.lang.Thread.run(Thread.java:844)
FAILED: org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.testNodeLost
Error Message:
Stack Trace:
java.util.concurrent.TimeoutException
at __randomizedtesting.SeedInfo.seed([8A7B7719C587386D:356EB9E7466D5DEB]:0)
at org.apache.solr.cloud.autoscaling.sim.SimSolrCloudTestCase.waitForState(SimSolrCloudTestCase.java:271)
at org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.doTestNodeLost(TestLargeCluster.java:453)
at org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.testNodeLost(TestLargeCluster.java:295)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:564)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.base/java.lang.Thread.run(Thread.java:844)
Build Log:
[...truncated 13657 lines...]
[junit4] Suite: org.apache.solr.cloud.autoscaling.sim.TestLargeCluster
[junit4] 2> 2151391 INFO (SUITE-TestLargeCluster-seed#[8A7B7719C587386D]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.sim.TestLargeCluster_8A7B7719C587386D-001/init-core-data-001
[junit4] 2> 2151392 INFO (SUITE-TestLargeCluster-seed#[8A7B7719C587386D]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 2151392 INFO (SUITE-TestLargeCluster-seed#[8A7B7719C587386D]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason="", ssl=0.0/0.0, value=0.0/0.0, clientAuth=0.0/0.0)
[junit4] 2> 2151393 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
[junit4] 2> 2151393 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 0
[junit4] 2> 2151394 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10022_solr, 127.0.0.1:10018_solr, 127.0.0.1:10016_solr, 127.0.0.1:10021_solr, 127.0.0.1:10017_solr, 127.0.0.1:10020_solr, 127.0.0.1:10019_solr, 127.0.0.1:10023_solr]
[junit4] 2> 2151394 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion -1
[junit4] 2> 2151394 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 0
[junit4] 2> 2151394 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 2151394 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion 0
[junit4] 2> 2151394 DEBUG (ScheduledTrigger-8416-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 16
[junit4] 2> 2151400 INFO (TEST-TestLargeCluster.testBasic-seed#[8A7B7719C587386D]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testBasic
[junit4] 2> 2151400 DEBUG (TEST-TestLargeCluster.testBasic-seed#[8A7B7719C587386D]) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
[junit4] 2> 2151400 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
[junit4] 2> 2151400 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
[junit4] 2> 2151400 INFO (TEST-TestLargeCluster.testBasic-seed#[8A7B7719C587386D]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
[junit4] 2> 2151400 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 2151401 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Interrupted
[junit4] 2> java.lang.InterruptedException
[junit4] 2> at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1259)
[junit4] 2> at java.base/java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:317)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:154)
[junit4] 2> at java.base/java.lang.Thread.run(Thread.java:844)
[junit4] 2> 2151401 DEBUG (TEST-TestLargeCluster.testBasic-seed#[8A7B7719C587386D]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
[junit4] 2> 2151401 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
[junit4] 2> 2151401 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2
[junit4] 2> 2151401 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10082_solr, 127.0.0.1:10018_solr, 127.0.0.1:10049_solr, 127.0.0.1:10104_solr, 127.0.0.1:10020_solr, 127.0.0.1:10065_solr, 127.0.0.1:10021_solr, 127.0.0.1:10066_solr, 127.0.0.1:10035_solr, 127.0.0.1:10051_solr, 127.0.0.1:10019_solr, 127.0.0.1:10096_solr, 127.0.0.1:10022_solr, 127.0.0.1:10033_solr, 127.0.0.1:10105_solr, 127.0.0.1:10017_solr, 127.0.0.1:10047_solr, 127.0.0.1:10102_solr, 127.0.0.1:10095_solr, 127.0.0.1:10081_solr, 127.0.0.1:10084_solr, 127.0.0.1:10036_solr, 127.0.0.1:10052_solr, 127.0.0.1:10079_solr, 127.0.0.1:10098_solr, 127.0.0.1:10054_solr, 127.0.0.1:10099_solr, 127.0.0.1:10101_solr, 127.0.0.1:10071_solr, 127.0.0.1:10023_solr, 127.0.0.1:10076_solr, 127.0.0.1:10107_solr, 127.0.0.1:10055_solr, 127.0.0.1:10016_solr, 127.0.0.1:10070_solr, 127.0.0.1:10077_solr, 127.0.0.1:10038_solr, 127.0.0.1:10032_solr, 127.0.0.1:10108_solr, 127.0.0.1:10093_solr, 127.0.0.1:10061_solr, 127.0.0.1:10067_solr, 127.0.0.1:10086_solr, 127.0.0.1:10089_solr, 127.0.0.1:10045_solr, 127.0.0.1:10042_solr, 127.0.0.1:10029_solr, 127.0.0.1:10083_solr, 127.0.0.1:10114_solr, 127.0.0.1:10080_solr, 127.0.0.1:10048_solr, 127.0.0.1:10064_solr, 127.0.0.1:10111_solr, 127.0.0.1:10043_solr, 127.0.0.1:10057_solr, 127.0.0.1:10112_solr, 127.0.0.1:10074_solr, 127.0.0.1:10026_solr, 127.0.0.1:10060_solr, 127.0.0.1:10073_solr, 127.0.0.1:10058_solr, 127.0.0.1:10113_solr, 127.0.0.1:10090_solr, 127.0.0.1:10088_solr, 127.0.0.1:10027_solr, 127.0.0.1:10039_solr, 127.0.0.1:10025_solr, 127.0.0.1:10044_solr, 127.0.0.1:10028_solr, 127.0.0.1:10092_solr, 127.0.0.1:10041_solr, 127.0.0.1:10087_solr, 127.0.0.1:10030_solr, 127.0.0.1:10040_solr, 127.0.0.1:10085_solr, 127.0.0.1:10046_solr, 127.0.0.1:10068_solr, 127.0.0.1:10063_solr, 127.0.0.1:10115_solr, 127.0.0.1:10069_solr, 127.0.0.1:10110_solr, 127.0.0.1:10062_solr, 127.0.0.1:10024_solr, 127.0.0.1:10050_solr, 127.0.0.1:10075_solr, 127.0.0.1:10078_solr, 127.0.0.1:10097_solr, 127.0.0.1:10031_solr, 127.0.0.1:10034_solr, 127.0.0.1:10091_solr, 127.0.0.1:10094_solr, 127.0.0.1:10106_solr, 127.0.0.1:10100_solr, 127.0.0.1:10037_solr, 127.0.0.1:10109_solr, 127.0.0.1:10053_solr, 127.0.0.1:10059_solr, 127.0.0.1:10103_solr, 127.0.0.1:10072_solr, 127.0.0.1:10056_solr]
[junit4] 2> 2151401 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion -1
[junit4] 2> 2151401 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
[junit4] 2> 2151401 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 2151401 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
[junit4] 2> 2151402 DEBUG (ScheduledTrigger-8419-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2151422 DEBUG (ScheduledTrigger-8419-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2151442 DEBUG (ScheduledTrigger-8419-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2151463 DEBUG (ScheduledTrigger-8419-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2151483 DEBUG (ScheduledTrigger-8419-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2151503 DEBUG (simCloudManagerPool-8418-thread-1) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 2151503 DEBUG (simCloudManagerPool-8418-thread-1) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 3
[junit4] 2> 2151503 DEBUG (simCloudManagerPool-8418-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10082_solr, 127.0.0.1:10018_solr, 127.0.0.1:10049_solr, 127.0.0.1:10104_solr, 127.0.0.1:10020_solr, 127.0.0.1:10065_solr, 127.0.0.1:10021_solr, 127.0.0.1:10066_solr, 127.0.0.1:10035_solr, 127.0.0.1:10051_solr, 127.0.0.1:10019_solr, 127.0.0.1:10096_solr, 127.0.0.1:10022_solr, 127.0.0.1:10033_solr, 127.0.0.1:10105_solr, 127.0.0.1:10017_solr, 127.0.0.1:10047_solr, 127.0.0.1:10102_solr, 127.0.0.1:10095_solr, 127.0.0.1:10081_solr, 127.0.0.1:10084_solr, 127.0.0.1:10036_solr, 127.0.0.1:10052_solr, 127.0.0.1:10079_solr, 127.0.0.1:10098_solr, 127.0.0.1:10054_solr, 127.0.0.1:10099_solr, 127.0.0.1:10101_solr, 127.0.0.1:10071_solr, 127.0.0.1:10023_solr, 127.0.0.1:10076_solr, 127.0.0.1:10107_solr, 127.0.0.1:10055_solr, 127.0.0.1:10016_solr, 127.0.0.1:10070_solr, 127.0.0.1:10077_solr, 127.0.0.1:10038_solr, 127.0.0.1:10032_solr, 127.0.0.1:10108_solr, 127.0.0.1:10093_solr, 127.0.0.1:10061_solr, 127.0.0.1:10067_solr, 127.0.0.1:10086_solr, 127.0.0.1:10089_solr, 127.0.0.1:10045_solr, 127.0.0.1:10042_solr, 127.0.0.1:10029_solr, 127.0.0.1:10083_solr, 127.0.0.1:10114_solr, 127.0.0.1:10080_solr, 127.0.0.1:10048_solr, 127.0.0.1:10064_solr, 127.0.0.1:10111_solr, 127.0.0.1:10043_solr, 127.0.0.1:10057_solr, 127.0.0.1:10112_solr, 127.0.0.1:10074_solr, 127.0.0.1:10026_solr, 127.0.0.1:10060_solr, 127.0.0.1:10073_solr, 127.0.0.1:10058_solr, 127.0.0.1:10113_solr, 127.0.0.1:10090_solr, 127.0.0.1:10088_solr, 127.0.0.1:10027_solr, 127.0.0.1:10039_solr, 127.0.0.1:10025_solr, 127.0.0.1:10044_solr, 127.0.0.1:10028_solr, 127.0.0.1:10092_solr, 127.0.0.1:10041_solr, 127.0.0.1:10087_solr, 127.0.0.1:10030_solr, 127.0.0.1:10040_solr, 127.0.0.1:10085_solr, 127.0.0.1:10046_solr, 127.0.0.1:10068_solr, 127.0.0.1:10063_solr, 127.0.0.1:10115_solr, 127.0.0.1:10069_solr, 127.0.0.1:10110_solr, 127.0.0.1:10062_solr, 127.0.0.1:10024_solr, 127.0.0.1:10050_solr, 127.0.0.1:10075_solr, 127.0.0.1:10078_solr, 127.0.0.1:10097_solr, 127.0.0.1:10031_solr, 127.0.0.1:10034_solr, 127.0.0.1:10091_solr, 127.0.0.1:10094_solr, 127.0.0.1:10106_solr, 127.0.0.1:10100_solr, 127.0.0.1:10037_solr, 127.0.0.1:10109_solr, 127.0.0.1:10053_solr, 127.0.0.1:10059_solr, 127.0.0.1:10103_solr, 127.0.0.1:10072_solr, 127.0.0.1:10056_solr]
[junit4] 2> 2151503 DEBUG (ScheduledTrigger-8419-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2151503 DEBUG (simCloudManagerPool-8418-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10082_solr, 127.0.0.1:10018_solr, 127.0.0.1:10049_solr, 127.0.0.1:10104_solr, 127.0.0.1:10020_solr, 127.0.0.1:10065_solr, 127.0.0.1:10021_solr, 127.0.0.1:10066_solr, 127.0.0.1:10035_solr, 127.0.0.1:10051_solr, 127.0.0.1:10019_solr, 127.0.0.1:10096_solr, 127.0.0.1:10022_solr, 127.0.0.1:10033_solr, 127.0.0.1:10105_solr, 127.0.0.1:10017_solr, 127.0.0.1:10047_solr, 127.0.0.1:10102_solr, 127.0.0.1:10095_solr, 127.0.0.1:10081_solr, 127.0.0.1:10084_solr, 127.0.0.1:10036_solr, 127.0.0.1:10052_solr, 127.0.0.1:10079_solr, 127.0.0.1:10098_solr, 127.0.0.1:10054_solr, 127.0.0.1:10099_solr, 127.0.0.1:10101_solr, 127.0.0.1:10071_solr, 127.0.0.1:10023_solr, 127.0.0.1:10076_solr, 127.0.0.1:10107_solr, 127.0.0.1:10055_solr, 127.0.0.1:10016_solr, 127.0.0.1:10070_solr, 127.0.0.1:10077_solr, 127.0.0.1:10038_solr, 127.0.0.1:10032_solr, 127.0.0.1:10108_solr, 127.0.0.1:10093_solr, 127.0.0.1:10061_solr, 127.0.0.1:10067_solr, 127.0.0.1:10086_solr, 127.0.0.1:10089_solr, 127.0.0.1:10045_solr, 127.0.0.1:10042_solr, 127.0.0.1:10029_solr, 127.0.0.1:10083_solr, 127.0.0.1:10114_solr, 127.0.0.1:10080_solr, 127.0.0.1:10048_solr, 127.0.0.1:10064_solr, 127.0.0.1:10111_solr, 127.0.0.1:10043_solr, 127.0.0.1:10057_solr, 127.0.0.1:10112_solr, 127.0.0.1:10074_solr, 127.0.0.1:10026_solr, 127.0.0.1:10060_solr, 127.0.0.1:10073_solr, 127.0.0.1:10058_solr, 127.0.0.1:10113_solr, 127.0.0.1:10090_solr, 127.0.0.1:10088_solr, 127.0.0.1:10027_solr, 127.0.0.1:10039_solr, 127.0.0.1:10025_solr, 127.0.0.1:10044_solr, 127.0.0.1:10028_solr, 127.0.0.1:10092_solr, 127.0.0.1:10041_solr, 127.0.0.1:10087_solr, 127.0.0.1:10030_solr, 127.0.0.1:10040_solr, 127.0.0.1:10085_solr, 127.0.0.1:10046_solr, 127.0.0.1:10068_solr, 127.0.0.1:10063_solr, 127.0.0.1:10115_solr, 127.0.0.1:10069_solr, 127.0.0.1:10110_solr, 127.0.0.1:10062_solr, 127.0.0.1:10024_solr, 127.0.0.1:10050_solr, 127.0.0.1:10075_solr, 127.0.0.1:10078_solr, 127.0.0.1:10097_solr, 127.0.0.1:10031_solr, 127.0.0.1:10034_solr, 127.0.0.1:10091_solr, 127.0.0.1:10094_solr, 127.0.0.1:10106_solr, 127.0.0.1:10100_solr, 127.0.0.1:10037_solr, 127.0.0.1:10109_solr, 127.0.0.1:10053_solr, 127.0.0.1:10059_solr, 127.0.0.1:10103_solr, 127.0.0.1:10072_solr, 127.0.0.1:10056_solr]
[junit4] 2> 2151503 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
[junit4] 2> 2151504 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 2151504 DEBUG (ScheduledTrigger-8419-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2151504 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
[junit4] 2> 2151505 DEBUG (simCloudManagerPool-8418-thread-2) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 2151505 DEBUG (simCloudManagerPool-8418-thread-2) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 4
[junit4] 2> 2151505 DEBUG (simCloudManagerPool-8418-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10082_solr, 127.0.0.1:10018_solr, 127.0.0.1:10049_solr, 127.0.0.1:10104_solr, 127.0.0.1:10020_solr, 127.0.0.1:10065_solr, 127.0.0.1:10021_solr, 127.0.0.1:10066_solr, 127.0.0.1:10035_solr, 127.0.0.1:10051_solr, 127.0.0.1:10019_solr, 127.0.0.1:10096_solr, 127.0.0.1:10022_solr, 127.0.0.1:10033_solr, 127.0.0.1:10105_solr, 127.0.0.1:10017_solr, 127.0.0.1:10047_solr, 127.0.0.1:10102_solr, 127.0.0.1:10095_solr, 127.0.0.1:10081_solr, 127.0.0.1:10084_solr, 127.0.0.1:10036_solr, 127.0.0.1:10052_solr, 127.0.0.1:10079_solr, 127.0.0.1:10098_solr, 127.0.0.1:10054_solr, 127.0.0.1:10099_solr, 127.0.0.1:10101_solr, 127.0.0.1:10071_solr, 127.0.0.1:10023_solr, 127.0.0.1:10076_solr, 127.0.0.1:10107_solr, 127.0.0.1:10055_solr, 127.0.0.1:10016_solr, 127.0.0.1:10070_solr, 127.0.0.1:10077_solr, 127.0.0.1:10038_solr, 127.0.0.1:10032_solr, 127.0.0.1:10108_solr, 127.0.0.1:10093_solr, 127.0.0.1:10061_solr, 127.0.0.1:10067_solr, 127.0.0.1:10086_solr, 127.0.0.1:10089_solr, 127.0.0.1:10045_solr, 127.0.0.1:10042_solr, 127.0.0.1:10029_solr, 127.0.0.1:10083_solr, 127.0.0.1:10114_solr, 127.0.0.1:10080_solr, 127.0.0.1:10048_solr, 127.0.0.1:10064_solr, 127.0.0.1:10111_solr, 127.0.0.1:10043_solr, 127.0.0.1:10057_solr, 127.0.0.1:10112_solr, 127.0.0.1:10074_solr, 127.0.0.1:10026_solr, 127.0.0.1:10060_solr, 127.0.0.1:10073_solr, 127.0.0.1:10058_solr, 127.0.0.1:10113_solr, 127.0.0.1:10090_solr, 127.0.0.1:10088_solr, 127.0.0.1:10027_solr, 127.0.0.1:10039_solr, 127.0.0.1:10025_solr, 127.0.0.1:10044_solr, 127.0.0.1:10028_solr, 127.0.0.1:10092_solr, 127.0.0.1:10041_solr, 127.0.0.1:10087_solr, 127.0.0.1:10030_solr, 127.0.0.1:10040_solr, 127.0.0.1:10085_solr, 127.0.0.1:10046_solr, 127.0.0.1:10068_solr, 127.0.0.1:10063_solr, 127.0.0.1:10115_solr, 127.0.0.1:10069_solr, 127.0.0.1:10110_solr, 127.0.0.1:10062_solr, 127.0.0.1:10024_solr, 127.0.0.1:10050_solr, 127.0.0.1:10075_solr, 127.0.0.1:10078_solr, 127.0.0.1:10097_solr, 127.0.0.1:10031_solr, 127.0.0.1:10034_solr, 127.0.0.1:10091_solr, 127.0.0.1:10094_solr, 127.0.0.1:10106_solr, 127.0.0.1:10100_solr, 127.0.0.1:10037_solr, 127.0.0.1:10109_solr, 127.0.0.1:10053_solr, 127.0.0.1:10059_solr, 127.0.0.1:10103_solr, 127.0.0.1:10072_solr, 127.0.0.1:10056_solr]
[junit4] 2> 2151505 DEBUG (simCloudManagerPool-8418-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10082_solr, 127.0.0.1:10018_solr, 127.0.0.1:10049_solr, 127.0.0.1:10104_solr, 127.0.0.1:10020_solr, 127.0.0.1:10065_solr, 127.0.0.1:10021_solr, 127.0.0.1:10066_solr, 127.0.0.1:10035_solr, 127.0.0.1:10051_solr, 127.0.0.1:10019_solr, 127.0.0.1:10096_solr, 127.0.0.1:10022_solr, 127.0.0.1:10033_solr, 127.0.0.1:10105_solr, 127.0.0.1:10017_solr, 127.0.0.1:10047_solr, 127.0.0.1:10102_solr, 127.0.0.1:10095_solr, 127.0.0.1:10081_solr, 127.0.0.1:10084_solr, 127.0.0.1:10036_solr, 127.0.0.1:10052_solr, 127.0.0.1:10079_solr, 127.0.0.1:10098_solr, 127.0.0.1:10054_solr, 127.0.0.1:10099_solr, 127.0.0.1:10101_solr, 127.0.0.1:10071_solr, 127.0.0.1:10023_solr, 127.0.0.1:10076_solr, 127.0.0.1:10107_solr, 127.0.0.1:10055_solr, 127.0.0.1:10016_solr, 127.0.0.1:10070_solr, 127.0.0.1:10077_solr, 127.0.0.1:10038_solr, 127.0.0.1:10032_solr, 127.0.0.1:10108_solr, 127.0.0.1:10093_solr, 127.0.0.1:10061_solr, 127.0.0.1:10067_solr, 127.0.0.1:10086_solr, 127.0.0.1:10089_solr, 127.0.0.1:10045_solr, 127.0.0.1:10042_solr, 127.0.0.1:10029_solr, 127.0.0.1:10083_solr, 127.0.0.1:10114_solr, 127.0.0.1:10080_solr, 127.0.0.1:10048_solr, 127.0.0.1:10064_solr, 127.0.0.1:10111_solr, 127.0.0.1:10043_solr, 127.0.0.1:10057_solr, 127.0.0.1:10112_solr, 127.0.0.1:10074_solr, 127.0.0.1:10026_solr, 127.0.0.1:10060_solr, 127.0.0.1:10073_solr, 127.0.0.1:10058_solr, 127.0.0.1:10113_solr, 127.0.0.1:10090_solr, 127.0.0.1:10088_solr, 127.0.0.1:10027_solr, 127.0.0.1:10039_solr, 127.0.0.1:10025_solr, 127.0.0.1:10044_solr, 127.0.0.1:10028_solr, 127.0.0.1:10092_solr, 127.0.0.1:10041_solr, 127.0.0.1:10087_solr, 127.0.0.1:10030_solr, 127.0.0.1:10040_solr, 127.0.0.1:10085_solr, 127.0.0.1:10046_solr, 127.0.0.1:10068_solr, 127.0.0.1:10063_solr, 127.0.0.1:10115_solr, 127.0.0.1:10069_solr, 127.0.0.1:10110_solr, 127.0.0.1:10062_solr, 127.0.0.1:10024_solr, 127.0.0.1:10050_solr, 127.0.0.1:10075_solr, 127.0.0.1:10078_solr, 127.0.0.1:10097_solr, 127.0.0.1:10031_solr, 127.0.0.1:10034_solr, 127.0.0.1:10091_solr, 127.0.0.1:10094_solr, 127.0.0.1:10106_solr, 127.0.0.1:10100_solr, 127.0.0.1:10037_solr, 127.0.0.1:10109_solr, 127.0.0.1:10053_solr, 127.0.0.1:10059_solr, 127.0.0.1:10103_solr, 127.0.0.1:10072_solr, 127.0.0.1:10056_solr]
[junit4] 2> 2151506 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 4
[junit4] 2> 2151506 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 2151506 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion 4
[junit4] 2> 2151524 DEBUG (ScheduledTrigger-8419-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2151524 DEBUG (ScheduledTrigger-8419-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2151544 DEBUG (ScheduledTrigger-8419-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2151544 DEBUG (ScheduledTrigger-8419-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2151564 DEBUG (ScheduledTrigger-8419-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2151564 DEBUG (ScheduledTrigger-8419-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2151585 DEBUG (ScheduledTrigger-8419-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2151585 DEBUG (ScheduledTrigger-8419-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2151605 DEBUG (ScheduledTrigger-8419-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2151605 DEBUG (ScheduledTrigger-8419-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2151625 DEBUG (ScheduledTrigger-8419-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2151625 DEBUG (ScheduledTrigger-8419-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2151645 DEBUG (ScheduledTrigger-8419-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2151645 DEBUG (ScheduledTrigger-8419-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2151666 DEBUG (ScheduledTrigger-8419-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2151666 DEBUG (ScheduledTrigger-8419-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2151686 DEBUG (ScheduledTrigger-8419-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2151686 DEBUG (ScheduledTrigger-8419-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2151706 DEBUG (ScheduledTrigger-8419-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2151706 DEBUG (ScheduledTrigger-8419-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2151726 DEBUG (ScheduledTrigger-8419-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2151726 DEBUG (ScheduledTrigger-8419-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2151747 DEBUG (ScheduledTrigger-8419-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2151747 DEBUG (ScheduledTrigger-8419-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2151767 DEBUG (ScheduledTrigger-8419-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2151767 DEBUG (ScheduledTrigger-8419-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2151787 DEBUG (ScheduledTrigger-8419-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2151787 DEBUG (ScheduledTrigger-8419-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2151807 DEBUG (ScheduledTrigger-8419-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2151807 DEBUG (ScheduledTrigger-8419-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2151828 DEBUG (ScheduledTrigger-8419-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2151828 DEBUG (ScheduledTrigger-8419-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2151848 DEBUG (ScheduledTrigger-8419-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2151848 DEBUG (ScheduledTrigger-8419-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2151868 DEBUG (ScheduledTrigger-8419-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2151868 DEBUG (ScheduledTrigger-8419-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2151889 DEBUG (ScheduledTrigger-8419-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2151889 DEBUG (ScheduledTrigger-8419-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2151909 DEBUG (ScheduledTrigger-8419-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2151909 DEBUG (ScheduledTrigger-8419-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2151929 DEBUG (ScheduledTrigger-8419-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2151929 DEBUG (ScheduledTrigger-8419-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2151949 DEBUG (ScheduledTrigger-8419-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2151949 DEBUG (ScheduledTrigger-8419-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2151970 DEBUG (ScheduledTrigger-8419-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2151970 DEBUG (ScheduledTrigger-8419-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2151990 DEBUG (ScheduledTrigger-8419-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2151990 DEBUG (ScheduledTrigger-8419-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2152010 DEBUG (ScheduledTrigger-8419-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2152010 DEBUG (ScheduledTrigger-8419-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2152031 DEBUG (ScheduledTrigger-8419-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2152031 DEBUG (ScheduledTrigger-8419-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2152051 DEBUG (ScheduledTrigger-8419-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2152051 DEBUG (ScheduledTrigger-8419-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2152071 DEBUG (ScheduledTrigger-8419-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2152071 DEBUG (ScheduledTrigger-8419-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2152091 DEBUG (ScheduledTrigger-8419-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2152091 DEBUG (ScheduledTrigger-8419-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2152112 DEBUG (ScheduledTrigger-8419-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2152112 DEBUG (ScheduledTrigger-8419-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2152132 DEBUG (ScheduledTrigger-8419-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2152132 DEBUG (ScheduledTrigger-8419-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2152152 DEBUG (ScheduledTrigger-8419-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2152152 DEBUG (ScheduledTrigger-8419-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2152172 DEBUG (ScheduledTrigger-8419-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2152172 DEBUG (ScheduledTrigger-8419-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2152193 DEBUG (ScheduledTrigger-8419-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2152193 DEBUG (ScheduledTrigger-8419-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2152213 DEBUG (ScheduledTrigger-8419-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2152213 DEBUG (ScheduledTrigger-8419-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2152233 DEBUG (ScheduledTrigger-8419-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2152233 DEBUG (ScheduledTrigger-8419-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2152253 DEBUG (ScheduledTrigger-8419-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2152253 DEBUG (ScheduledTrigger-8419-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2152274 DEBUG (ScheduledTrigger-8419-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2152274 DEBUG (ScheduledTrigger-8419-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2152294 DEBUG (ScheduledTrigger-8419-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2152294 DEBUG (ScheduledTrigger-8419-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2152314 DEBUG (ScheduledTrigger-8419-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2152314 DEBUG (ScheduledTrigger-8419-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2152335 DEBUG (ScheduledTrigger-8419-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2152335 DEBUG (ScheduledTrigger-8419-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2152355 DEBUG (ScheduledTrigger-8419-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2152355 DEBUG (ScheduledTrigger-8419-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2152375 DEBUG (ScheduledTrigger-8419-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2152375 DEBUG (ScheduledTrigger-8419-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 2152377 INFO (TEST-TestLargeCluster.testBasic-seed#[8A7B7719C587386D]) [ ] o.a.s.c.a.s.TestLargeCluster Ready after 37704ms
[junit4] 2> 2152381 DEBUG (simCloudManagerPool-8418-thread-79) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testBasic / shard2: {"core_node18":{
[junit4] 2> "node_name":"127.0.0.1:10023_solr",
[junit4] 2> "core":"testBasic_shard2_replica_n18",
[junit4] 2> "leader":"true",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInBytes":123450000,
[junit4] 2> "shard":"shard2",
[junit4] 2> "collection":"testBasic"}}
[junit4] 2> 2152381 DEBUG (simCloudManagerPool-8418-thread-80) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testBasic / shard3: {"core_node39":{
[junit4] 2> "node_name":"127.0.0.1:10104_solr",
[junit4] 2> "core":"testBasic_shard3_replica_t39",
[junit4] 2> "leader":"true",
[junit4] 2> "state":"active",
[junit4] 2> "type":"TLOG",
[junit4] 2> "INDEX.sizeInBytes":123450000,
[junit4] 2> "shard":"shard3",
[junit4] 2> "collection":"testBasic"}}
[junit4] 2> 2152381 DEBUG (simCloudManagerPool-8418-thread-83) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testBasic / shard1: {"core_node1":{
[junit4] 2> "node_name":"127.0.0.1:10101_solr",
[junit4] 2> "core":"testBasic_shard1_replica_n1",
[junit4] 2> "leader":"true",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInBytes":123450000,
[junit4] 2> "shard":"shard1",
[junit4] 2> "collection":"testBasic"}}
[junit4] 2> 2152381 DEBUG (simCloudManagerPool-8418-thread-81) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testBasic / shard4: {"core_node54":{
[junit4] 2> "node_name":"127.0.0.1:10064_solr",
[junit4] 2> "core":"testBasic_shard4_replica_t54",
[junit4] 2> "leader":"true",
[junit4] 2> "state":"active",
[junit4] 2> "type":"TLOG",
[junit4] 2> "INDEX.sizeInBytes":123450000,
[junit4] 2> "shard":"shard4",
[junit4] 2> "collection":"testBasic"}}
[junit4] 2> 2152382 DEBUG (simCloudManagerPool-8418-thread-82) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testBasic / shard5: {"core_node65":{
[junit4] 2> "node_name":"127.0.0.1:10035_solr",
[junit4] 2> "core":"testBasic_shard5_replica_n65",
[junit4] 2> "leader":"true",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInBytes":123450000,
[junit4] 2> "shard":"shard5",
[junit4] 2> "collection":"testBasic"}}
[junit4] 2> 2152395 DEBUG (ScheduledTrigger-8419-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 96
[junit4] 2> 2152395 DEBUG (ScheduledTrigger-8419-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 96
[junit4] 2> 2152396 DEBUG (ScheduledTrigger-8419-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10019_solr
[junit4] 2> 2152396 DEBUG (ScheduledTrigger-8419-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10019_solr
[junit4] 2> 2152396 DEBUG (ScheduledTrigger-8419-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10105_solr
[junit4] 2> 2152396 DEBUG (ScheduledTrigger-8419-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10105_solr
[junit4] 2> 2152396 DEBUG (ScheduledTrigger-8419-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10052_solr
[junit4] 2> 2152396 DEBUG (ScheduledTrigger-8419-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10026_solr
[junit4] 2> 2152396 DEBUG (ScheduledTrigger-8419-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10052_solr
[junit4] 2> 2152396 DEBUG (ScheduledTrigger-8419-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10026_solr
[junit4] 2> 2152416 DEBUG (ScheduledTrigger-8419-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 92
[junit4] 2> 2152416 DEBUG (ScheduledTrigger-8419-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10022_solr
[junit4] 2> 2152416 DEBUG (ScheduledTrigger-8419-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10036_solr
[junit4] 2> 2152416 DEBUG (ScheduledTrigger-8419-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10071_solr
[junit4] 2> 2152416 DEBUG (ScheduledTrigger-8419-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10076_solr
[junit4] 2> 2152416 DEBUG (ScheduledTrigger-8419-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 92
[junit4] 2> 2152416 DEBUG (ScheduledTrigger-8419-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10022_solr
[junit4] 2> 2152416 DEBUG (ScheduledTrigger-8419-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10036_solr
[junit4] 2> 2152416 DEBUG (ScheduledTrigger-8419-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10071_solr
[junit4] 2> 2152416 DEBUG (ScheduledTrigger-8419-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10076_solr
[junit4] 2> 2152436 DEBUG (ScheduledTrigger-8419-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 92
[junit4] 2> 2152437 DEBUG (ScheduledTrigger-8419-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 92
[junit4] 2> 2152457 DEBUG (ScheduledTrigger-8419-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 92
[junit4] 2> 2152457 DEBUG (ScheduledTrigger-8419-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 92
[junit4] 2> 2152477 DEBUG (ScheduledTrigger-8419-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 92
[junit4] 2> 2152477 DEBUG (ScheduledTrigger-8419-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 92
[junit4] 2> 2152497 DEBUG (ScheduledTrigger-8419-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 92
[junit4] 2> 2152497 DEBUG (ScheduledTrigger-8419-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 92
[junit4] 2> 2152497 DEBUG (ScheduledTrigger-8419-thread-2) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:10052_solr, 127.0.0.1:10105_solr, 127.0.0.1:10026_solr, 127.0.0.1:10019_solr]
[junit4] 2> 2152497 DEBUG (ScheduledTrigger-8419-thread-2) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
[junit4] 2> "id":"7c25ef310339Tbb8x7tcy164rfweb08jol5m0l",
[junit4] 2> "source":"node_lost_trigger",
[junit4] 2> "eventTime":136502368600889,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 136502368600889,
[junit4] 2> 136502363258989,
[junit4] 2> 136502370026839,
[junit4] 2> 136502359925389],
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10052_solr",
[junit4] 2> "127.0.0.1:10105_solr",
[junit4] 2> "127.0.0.1:10026_solr",
[junit4] 2> "127.0.0.1:10019_solr"]}}
[junit4] 2> 2152498 DEBUG (ScheduledTrigger-8419-thread-2) [ ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_lost_trigger]
[junit4] 2> 2152498 DEBUG (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
[junit4] 2> "id":"7c25ef310339Tbb8x7tcy164rfweb08jol5m0l",
[junit4] 2> "source":"node_lost_trigger",
[junit4] 2> "eventTime":136502368600889,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 136502368600889,
[junit4] 2> 136502363258989,
[junit4] 2> 136502370026839,
[junit4] 2> 136502359925389],
[junit4] 2> "_enqueue_time_":136507460497839,
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10052_solr",
[junit4] 2> "127.0.0.1:10105_solr",
[junit4] 2> "127.0.0.1:10026_solr",
[junit4] 2> "127.0.0.1:10019_solr"]}}
[junit4] 2> 2152499 DEBUG (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.a.ComputePlanAction -- processing event: {
[junit4] 2> "id":"7c25ef310339Tbb8x7tcy164rfweb08jol5m0l",
[junit4] 2> "source":"node_lost_trigger",
[junit4] 2> "eventTime":136502368600889,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 136502368600889,
[junit4] 2> 136502363258989,
[junit4] 2> 136502370026839,
[junit4] 2> 136502359925389],
[junit4] 2> "_enqueue_time_":136507460497839,
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10052_solr",
[junit4] 2> "127.0.0.1:10105_solr",
[junit4] 2> "127.0.0.1:10026_solr",
[junit4] 2> "127.0.0.1:10019_solr"]}} with context properties: {BEFORE_ACTION=[compute]}
[junit4] 2> 2152511 INFO (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10056_solr&inPlaceMove=true&replica=core_node53
[junit4] 2> 2152518 INFO (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10072_solr&inPlaceMove=true&replica=core_node40
[junit4] 2> 2152521 INFO (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10103_solr&inPlaceMove=true&replica=core_node66
[junit4] 2> 2152523 INFO (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10059_solr&inPlaceMove=true&replica=core_node26
[junit4] 2> 2152523 INFO (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 136508721 sessionWrapper.createTime 136507569037789, this.sessionWrapper.createTime 136507569037789
[junit4] 2> 2152523 DEBUG (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.a.ExecutePlanAction -- processing event: {
[junit4] 2> "id":"7c25ef310339Tbb8x7tcy164rfweb08jol5m0l",
[junit4] 2> "source":"node_lost_trigger",
[junit4] 2> "eventTime":136502368600889,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 136502368600889,
[junit4] 2> 136502363258989,
[junit4] 2> 136502370026839,
[junit4] 2> 136502359925389],
[junit4] 2> "_enqueue_time_":136507460497839,
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10052_solr",
[junit4] 2> "127.0.0.1:10105_solr",
[junit4] 2> "127.0.0.1:10026_solr",
[junit4] 2> "127.0.0.1:10019_solr"]}} with context properties: {operations=[org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@75c90a70, org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@7b0eeab9, org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@7f316100, org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@3a33e082], AFTER_ACTION=[compute], BEFORE_ACTION=[compute, execute]}
[junit4] 2> 2152523 DEBUG (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.a.ExecutePlanAction Executing operation: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10056_solr&inPlaceMove=true&replica=core_node53
[junit4] 2> 2152624 DEBUG (simCloudManagerPool-8418-thread-120) [ ] o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node77":{
[junit4] 2> "core":"testBasic_shard4_replica_t76",
[junit4] 2> "shard":"shard4",
[junit4] 2> "collection":"testBasic",
[junit4] 2> "node_name":"127.0.0.1:10056_solr",
[junit4] 2> "type":"TLOG"}}
[junit4] 2> 2152736 DEBUG (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.a.ExecutePlanAction Executing operation: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10072_solr&inPlaceMove=true&replica=core_node40
[junit4] 2> 2152836 DEBUG (simCloudManagerPool-8418-thread-128) [ ] o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node79":{
[junit4] 2> "core":"testBasic_shard3_replica_t78",
[junit4] 2> "shard":"shard3",
[junit4] 2> "collection":"testBasic",
[junit4] 2> "node_name":"127.0.0.1:10072_solr",
[junit4] 2> "type":"TLOG"}}
[junit4] 2> 2152948 DEBUG (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.a.ExecutePlanAction Executing operation: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10103_solr&inPlaceMove=true&replica=core_node66
[junit4] 2> 2153049 DEBUG (simCloudManagerPool-8418-thread-136) [ ] o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node81":{
[junit4] 2> "core":"testBasic_shard5_replica_t80",
[junit4] 2> "shard":"shard5",
[junit4] 2> "collection":"testBasic",
[junit4] 2> "node_name":"127.0.0.1:10103_solr",
[junit4] 2> "type":"TLOG"}}
[junit4] 2> 2153161 DEBUG (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.a.ExecutePlanAction Executing operation: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10059_solr&inPlaceMove=true&replica=core_node26
[junit4] 2> 2153261 DEBUG (simCloudManagerPool-8418-thread-144) [ ] o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node83":{
[junit4] 2> "core":"testBasic_shard2_replica_p82",
[junit4] 2> "shard":"shard2",
[junit4] 2> "collection":"testBasic",
[junit4] 2> "node_name":"127.0.0.1:10059_solr",
[junit4] 2> "type":"PULL"}}
[junit4] 2> 2153373 DEBUG (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms
[junit4] 2> 2153373 DEBUG (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_lost_trigger after 100ms
[junit4] 2> 2153373 DEBUG (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 43728 ms for event id=7c25ef310339Tbb8x7tcy164rfweb08jol5m0l
[junit4] 2> 2153473 DEBUG (ScheduledTrigger-8419-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 92
[junit4] 2> 2153473 DEBUG (ScheduledTrigger-8419-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 92
[junit4] 2> 2153473 DEBUG (ScheduledTrigger-8419-thread-1) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:10052_solr, 127.0.0.1:10022_solr, 127.0.0.1:10105_solr, 127.0.0.1:10026_solr, 127.0.0.1:10071_solr, 127.0.0.1:10019_solr, 127.0.0.1:10036_solr, 127.0.0.1:10076_solr]
[junit4] 2> 2153473 DEBUG (ScheduledTrigger-8419-thread-3) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:10022_solr, 127.0.0.1:10071_solr, 127.0.0.1:10036_solr, 127.0.0.1:10076_solr]
[junit4] 2> 2153473 DEBUG (ScheduledTrigger-8419-thread-3) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
[junit4] 2> "id":"7c262c72b8ffTbb8x7tcy164rfweb08jol5m0p",
[junit4] 2> "source":"node_lost_trigger",
[junit4] 2> "eventTime":136503396317439,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 136503396317439,
[junit4] 2> 136503401186889,
[junit4] 2> 136503397793289,
[junit4] 2> 136503402846389],
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10022_solr",
[junit4] 2> "127.0.0.1:10071_solr",
[junit4] 2> "127.0.0.1:10036_solr",
[junit4] 2> "127.0.0.1:10076_solr"]}}
[junit4] 2> 2153473 DEBUG (ScheduledTrigger-8419-thread-1) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
[junit4] 2> "id":"7c25eefb446fTbb8x7tcy164rfweb08jol5m0o",
[junit4] 2> "source":".auto_add_replicas",
[junit4] 2> "eventTime":136502365078639,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 136502365078639,
[junit4] 2> 136503382704989,
[junit4] 2> 136502361399939,
[junit4] 2> 136502366679539,
[junit4] 2> 136503385718139,
[junit4] 2> 136502358192739,
[junit4] 2> 136503384267139,
[junit4] 2> 136503387178989],
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10052_solr",
[junit4] 2> "127.0.0.1:10022_solr",
[junit4] 2> "127.0.0.1:10105_solr",
[junit4] 2> "127.0.0.1:10026_solr",
[junit4] 2> "127.0.0.1:10071_solr",
[junit4] 2> "127.0.0.1:10019_solr",
[junit4] 2> "127.0.0.1:10036_solr",
[junit4] 2> "127.0.0.1:10076_solr"]}}
[junit4] 2> 2153473 DEBUG (ScheduledTrigger-8419-thread-3) [ ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_lost_trigger]
[junit4] 2> 2153474 DEBUG (ScheduledTrigger-8419-thread-1) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger processor for lost nodes: [127.0.0.1:10052_solr, 127.0.0.1:10022_solr, 127.0.0.1:10105_solr, 127.0.0.1:10026_solr, 127.0.0.1:10071_solr, 127.0.0.1:10019_solr, 127.0.0.1:10036_solr, 127.0.0.1:10076_solr] is not ready, will try later
[junit4] 2> 2153474 DEBUG (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
[junit4] 2> "id":"7c262c72b8ffTbb8x7tcy164rfweb08jol5m0p",
[junit4] 2> "source":"node_lost_trigger",
[junit4] 2> "eventTime":136503396317439,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 136503396317439,
[junit4] 2> 136503401186889,
[junit4] 2> 136503397793289,
[junit4] 2> 136503402846389],
[junit4] 2> "_enqueue_time_":136556253282539,
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10022_solr",
[junit4] 2> "127.0.0.1:10071_solr",
[junit4] 2> "127.0.0.1:10036_solr",
[junit4] 2> "127.0.0.1:10076_solr"]}}
[junit4] 2> 2153474 DEBUG (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.a.ComputePlanAction -- processing event: {
[junit4] 2> "id":"7c262c72b8ffTbb8x7tcy164rfweb08jol5m0p",
[junit4] 2> "source":"node_lost_trigger",
[junit4] 2> "eventTime":136503396317439,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 136503396317439,
[junit4] 2> 136503401186889,
[junit4] 2> 136503397793289,
[junit4] 2> 136503402846389],
[junit4] 2> "_enqueue_time_":136556253282539,
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10022_solr",
[junit4] 2> "127.0.0.1:10071_solr",
[junit4] 2> "127.0.0.1:10036_solr",
[junit4] 2> "127.0.0.1:10076_solr"]}} with context properties: {BEFORE_ACTION=[compute]}
[junit4] 2> 2153482 INFO (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10053_solr&inPlaceMove=true&replica=core_node70
[junit4] 2> 2153487 INFO (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10109_solr&inPlaceMove=true&replica=core_node17
[junit4] 2> 2153490 INFO (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10037_solr&inPlaceMove=true&replica=core_node48
[junit4] 2> 2153492 INFO (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10100_solr&inPlaceMove=true&replica=core_node23
[junit4] 2> 2153492 INFO (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 136557194 sessionWrapper.createTime 136556333599339, this.sessionWrapper.createTime 136556333599339
[junit4] 2> 2153493 DEBUG (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.a.ExecutePlanAction -- processing event: {
[junit4] 2> "id":"7c262c72b8ffTbb8x7tcy164rfweb08jol5m0p",
[junit4] 2> "source":"node_lost_trigger",
[junit4] 2> "eventTime":136503396317439,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 136503396317439,
[junit4] 2> 136503401186889,
[junit4] 2> 136503397793289,
[junit4] 2> 136503402846389],
[junit4] 2> "_enqueue_time_":136556253282539,
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10022_solr",
[junit4] 2> "127.0.0.1:10071_solr",
[junit4] 2> "127.0.0.1:10036_solr",
[junit4] 2> "127.0.0.1:10076_solr"]}} with context properties: {operations=[org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@37a7a747, org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@4b5c2da8, org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@3d0e4875, org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@5045cd61], AFTER_ACTION=[compute], BEFORE_ACTION=[compute, execute]}
[junit4] 2> 2153493 DEBUG (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.a.ExecutePlanAction Executing operation: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10053_solr&inPlaceMove=true&replica=core_node70
[junit4] 2> 2153593 DEBUG (simCloudManagerPool-8418-thread-155) [ ] o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node85":{
[junit4] 2> "core":"testBasic_shard5_replica_t84",
[junit4] 2> "shard":"shard5",
[junit4] 2> "collection":"testBasic",
[junit4] 2> "node_name":"127.0.0.1:10053_solr",
[junit4] 2> "type":"TLOG"}}
[junit4] 2> 2153705 DEBUG (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.a.ExecutePlanAction Executing operation: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10109_solr&inPlaceMove=true&replica=core_node17
[junit4] 2> 2153805 DEBUG (simCloudManagerPool-8418-thread-163) [ ] o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node87":{
[junit4] 2> "core":"testBasic_shard2_replica_n86",
[junit4] 2> "shard":"shard2",
[junit4] 2> "collection":"testBasic",
[junit4] 2> "node_name":"127.0.0.1:10109_solr",
[junit4] 2> "type":"NRT"}}
[junit4] 2> 2153917 DEBUG (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.a.ExecutePlanAction Executing operation: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10037_solr&inPlaceMove=true&replica=core_node48
[junit4] 2> 2154017 DEBUG (simCloudManagerPool-8418-thread-171) [ ] o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node89":{
[junit4] 2> "core":"testBasic_shard4_replica_n88",
[junit4] 2> "shard":"shard4",
[junit4] 2> "collection":"testBasic",
[junit4] 2> "node_name":"127.0.0.1:10037_solr",
[junit4] 2> "type":"NRT"}}
[junit4] 2> 2154129 DEBUG (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.a.ExecutePlanAction Executing operation: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10100_solr&inPlaceMove=true&replica=core_node23
[junit4] 2> 2154230 DEBUG (simCloudManagerPool-8418-thread-179) [ ] o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node91":{
[junit4] 2> "core":"testBasic_shard2_replica_t90",
[junit4] 2> "shard":"shard2",
[junit4] 2> "collection":"testBasic",
[junit4] 2> "node_name":"127.0.0.1:10100_solr",
[junit4] 2> "type":"TLOG"}}
[junit4] 2> 2154341 INFO (TEST-TestLargeCluster.testBasic-seed#[8A7B7719C587386D]) [ ] o.a.s.c.a.s.TestLargeCluster Ready after 96333ms
[junit4] 2> 2154341 DEBUG (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms
[junit4] 2> 2154342 DEBUG (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_lost_trigger after 100ms
[junit4] 2> 2154342 INFO (TEST-TestLargeCluster.testBasic-seed#[8A7B7719C587386D]) [ ] o.a.s.c.a.s.TestLargeCluster OP COUNTS: {CREATE=1, DELETESTATUS=8, MOVEREPLICA=8, REQUESTSTATUS=8, autoscaling=2, update=5}
[junit4] 2> 2154342 DEBUG (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 43379 ms for event id=7c262c72b8ffTbb8x7tcy164rfweb08jol5m0p
[junit4] 2> 2154345 DEBUG (simCloudManagerPool-8418-thread-8) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testBasic / shard5: {"core_node64":{
[junit4] 2> "node_name":"127.0.0.1:10047_solr",
[junit4] 2> "core":"testBasic_shard5_replica_n64",
[junit4] 2> "leader":"true",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInBytes":123450000,
[junit4] 2> "shard":"shard5",
[junit4] 2> "collection":"testBasic"}}
[junit4] 2> 2154442 DEBUG (ScheduledTrigger-8419-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 82
[junit4] 2> 2154442 DEBUG (ScheduledTrigger-8419-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 82
[junit4] 2> 2154442 DEBUG (ScheduledTrigger-8419-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10018_solr
[junit4] 2> 2154442 DEBUG (ScheduledTrigger-8419-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10018_solr
[junit4] 2> 2154442 DEBUG (ScheduledTrigger-8419-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10049_solr
[junit4] 2> 2154442 DEBUG (ScheduledTrigger-8419-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10049_solr
[junit4] 2> 2154442 DEBUG (ScheduledTrigger-8419-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10074_solr
[junit4] 2> 2154442 DEBUG (ScheduledTrigger-8419-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10074_solr
[junit4] 2> 2154442 DEBUG (ScheduledTrigger-8419-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10113_solr
[junit4] 2> 2154442 DEBUG (ScheduledTrigger-8419-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10113_solr
[junit4] 2> 2154442 DEBUG (ScheduledTrigger-8419-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10035_solr
[junit4] 2> 2154442 DEBUG (ScheduledTrigger-8419-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10035_solr
[junit4] 2> 2154442 DEBUG (ScheduledTrigger-8419-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10087_solr
[junit4] 2> 2154442 DEBUG (ScheduledTrigger-8419-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10087_solr
[junit4] 2> 2154442 DEBUG (ScheduledTrigger-8419-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10085_solr
[junit4] 2> 2154442 DEBUG (ScheduledTrigger-8419-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10085_solr
[junit4] 2> 2154442 DEBUG (ScheduledTrigger-8419-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10046_solr
[junit4] 2> 2154442 DEBUG (ScheduledTrigger-8419-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10046_solr
[junit4] 2> 2154442 DEBUG (ScheduledTrigger-8419-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10038_solr
[junit4] 2> 2154442 DEBUG (ScheduledTrigger-8419-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10038_solr
[junit4] 2> 2154442 DEBUG (ScheduledTrigger-8419-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10086_solr
[junit4] 2> 2154442 DEBUG (ScheduledTrigger-8419-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10086_solr
[junit4] 2> 2154442 DEBUG (ScheduledTrigger-8419-thread-2) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:10022_solr, 127.0.0.1:10105_solr, 127.0.0.1:10026_solr, 127.0.0.1:10071_solr, 127.0.0.1:10036_solr, 127.0.0.1:10076_solr, 127.0.0.1:10052_solr, 127.0.0.1:10019_solr]
[junit4] 2> 2154443 DEBUG (ScheduledTrigger-8419-thread-2) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
[junit4] 2> "id":"7c262ba3035dTbb8x7tcy164rfweb08jol5m0t",
[junit4] 2> "source":".auto_add_replicas",
[junit4] 2> "eventTime":136503382704989,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 136503382704989,
[junit4] 2> 136502361399939,
[junit4] 2> 136502366679539,
[junit4] 2> 136503385718139,
[junit4] 2> 136503384267139,
[junit4] 2> 136503387178989,
[junit4] 2> 136502365078639,
[junit4] 2> 136502358192739],
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10022_solr",
[junit4] 2> "127.0.0.1:10105_solr",
[junit4] 2> "127.0.0.1:10026_solr",
[junit4] 2> "127.0.0.1:10071_solr",
[junit4] 2> "127.0.0.1:10036_solr",
[junit4] 2> "127.0.0.1:10076_solr",
[junit4] 2> "127.0.0.1:10052_solr",
[junit4] 2> "127.0.0.1:10019_solr"]}}
[junit4] 2> 2154443 DEBUG (ScheduledTrigger-8419-thread-2) [ ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_lost_trigger]
[junit4] 2> 2154443 DEBUG (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
[junit4] 2> "id":"7c262ba3035dTbb8x7tcy164rfweb08jol5m0t",
[junit4] 2> "source":".auto_add_replicas",
[junit4] 2> "eventTime":136503382704989,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 136503382704989,
[junit4] 2> 136502361399939,
[junit4] 2> 136502366679539,
[junit4] 2> 136503385718139,
[junit4] 2> 136503384267139,
[junit4] 2> 136503387178989,
[junit4] 2> 136502365078639,
[junit4] 2> 136502358192739],
[junit4] 2> "_enqueue_time_":136604710274189,
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10022_solr",
[junit4] 2> "127.0.0.1:10105_solr",
[junit4] 2> "127.0.0.1:10026_solr",
[junit4] 2> "127.0.0.1:10071_solr",
[junit4] 2> "127.0.0.1:10036_solr",
[junit4] 2> "127.0.0.1:10076_solr",
[junit4] 2> "127.0.0.1:10052_solr",
[junit4] 2> "127.0.0.1:10019_solr"]}}
[junit4] 2> 2154443 DEBUG (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.a.ComputePlanAction -- processing event: {
[junit4] 2> "id":"7c262ba3035dTbb8x7tcy164rfweb08jol5m0t",
[junit4] 2> "source":".auto_add_replicas",
[junit4] 2> "eventTime":136503382704989,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 136503382704989,
[junit4] 2> 136502361399939,
[junit4] 2> 136502366679539,
[junit4] 2> 136503385718139,
[junit4] 2> 136503384267139,
[junit4] 2> 136503387178989,
[junit4] 2> 136502365078639,
[junit4] 2> 136502358192739],
[junit4] 2> "_enqueue_time_":136604710274189,
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10022_solr",
[junit4] 2> "127.0.0.1:10105_solr",
[junit4] 2> "127.0.0.1:10026_solr",
[junit4] 2> "127.0.0.1:10071_solr",
[junit4] 2> "127.0.0.1:10036_solr",
[junit4] 2> "127.0.0.1:10076_solr",
[junit4] 2> "127.0.0.1:10052_solr",
[junit4] 2> "127.0.0.1:10019_solr"]}} with context properties: {BEFORE_ACTION=[auto_add_replicas_plan]}
[junit4] 2> 2154445 INFO (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 136604805 sessionWrapper.createTime 136604799251589, this.sessionWrapper.createTime 136604799251589
[junit4] 2> 2154445 DEBUG (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.a.ExecutePlanAction -- processing event: {
[junit4] 2> "id":"7c262ba3035dTbb8x7tcy164rfweb08jol5m0t",
[junit4] 2> "source":".auto_add_replicas",
[junit4] 2> "eventTime":136503382704989,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 136503382704989,
[junit4] 2> 136502361399939,
[junit4] 2> 136502366679539,
[junit4] 2> 136503385718139,
[junit4] 2> 136503384267139,
[junit4] 2> 136503387178989,
[junit4] 2> 136502365078639,
[junit4] 2> 136502358192739],
[junit4] 2> "_enqueue_time_":136604710274189,
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10022_solr",
[junit4] 2> "127.0.0.1:10105_solr",
[junit4] 2> "127.0.0.1:10026_solr",
[junit4] 2> "127.0.0.1:10071_solr",
[junit4] 2> "127.0.0.1:10036_solr",
[junit4] 2> "127.0.0.1:10076_solr",
[junit4] 2> "127.0.0.1:10052_solr",
[junit4] 2> "127.0.0.1:10019_solr"]}} with context properties: {AFTER_ACTION=[auto_add_replicas_plan], BEFORE_ACTION=[auto_add_replicas_plan, execute_plan]}
[junit4] 2> 2154445 INFO (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.a.ExecutePlanAction No operations to execute for event: {
[junit4] 2> "id":"7c262ba3035dTbb8x7tcy164rfweb08jol5m0t",
[junit4] 2> "source":".auto_add_replicas",
[junit4] 2> "eventTime":136503382704989,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 136503382704989,
[junit4] 2> 136502361399939,
[junit4] 2> 136502366679539,
[junit4] 2> 136503385718139,
[junit4] 2> 136503384267139,
[junit4] 2> 136503387178989,
[junit4] 2> 136502365078639,
[junit4] 2> 136502358192739],
[junit4] 2> "_enqueue_time_":136604710274189,
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10022_solr",
[junit4] 2> "127.0.0.1:10105_solr",
[junit4] 2> "127.0.0.1:10026_solr",
[junit4] 2> "127.0.0.1:10071_solr",
[junit4] 2> "127.0.0.1:10036_solr",
[junit4] 2> "127.0.0.1:10076_solr",
[junit4] 2> "127.0.0.1:10052_solr",
[junit4] 2> "127.0.0.1:10019_solr"]}}
[junit4] 2> 2154445 DEBUG (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms
[junit4] 2> 2154445 DEBUG (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_lost_trigger after 100ms
[junit4] 2> 2154445 DEBUG (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 115 ms for event id=7c262ba3035dTbb8x7tcy164rfweb08jol5m0t
[junit4] 2> 2154545 DEBUG (ScheduledTrigger-8419-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 82
[junit4] 2> 2154545 DEBUG (ScheduledTrigger-8419-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 82
[junit4] 2> 2154545 DEBUG (ScheduledTrigger-8419-thread-1) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:10085_solr, 127.0.0.1:10018_solr, 127.0.0.1:10049_solr, 127.0.0.1:10087_solr, 127.0.0.1:10074_solr, 127.0.0.1:10046_solr, 127.0.0.1:10113_solr, 127.0.0.1:10035_solr, 127.0.0.1:10038_solr, 127.0.0.1:10086_solr]
[junit4] 2> 2154546 DEBUG (ScheduledTrigger-8419-thread-1) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
[junit4] 2> "id":"7c3dc1fda675Tbb8x7tcy164rfweb08jol5m0w",
[junit4] 2> "source":"node_lost_trigger",
[junit4] 2> "eventTime":136604689475189,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 136604689475189,
[junit4] 2> 136604668565089,
[junit4] 2> 136604671986439,
[junit4] 2> 136604684703439,
[junit4] 2> 136604675170889,
[junit4] 2> 136604692307289,
[junit4] 2> 136604678305839,
[junit4] 2> 136604681461739,
[junit4] 2> 136604695299239,
[junit4] 2> 136604698257039],
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10085_solr",
[junit4] 2> "127.0.0.1:10018_solr",
[junit4] 2> "127.0.0.1:10049_solr",
[junit4] 2> "127.0.0.1:10087_solr",
[junit4] 2> "127.0.0.1:10074_solr",
[junit4] 2> "127.0.0.1:10046_solr",
[junit4] 2> "127.0.0.1:10113_solr",
[junit4] 2> "127.0.0.1:10035_solr",
[junit4] 2> "127.0.0.1:10038_solr",
[junit4] 2> "127.0.0.1:10086_solr"]}}
[junit4] 2> 2154546 DEBUG (ScheduledTrigger-8419-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_lost_trigger]
[junit4] 2> 2154546 DEBUG (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
[junit4] 2> "id":"7c3dc1fda675Tbb8x7tcy164rfweb08jol5m0w",
[junit4] 2> "source":"node_lost_trigger",
[junit4] 2> "eventTime":136604689475189,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 136604689475189,
[junit4] 2> 136604668565089,
[junit4] 2> 136604671986439,
[junit4] 2> 136604684703439,
[junit4] 2> 136604675170889,
[junit4] 2> 136604692307289,
[junit4] 2> 136604678305839,
[junit4] 2> 136604681461739,
[junit4] 2> 136604695299239,
[junit4] 2> 136604698257039],
[junit4] 2> "_enqueue_time_":136609861116439,
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10085_solr",
[junit4] 2> "127.0.0.1:10018_solr",
[junit4] 2> "127.0.0.1:10049_solr",
[junit4] 2> "127.0.0.1:10087_solr",
[junit4] 2> "127.0.0.1:10074_solr",
[junit4] 2> "127.0.0.1:10046_solr",
[junit4] 2> "127.0.0.1:10113_solr",
[junit4] 2> "127.0.0.1:10035_solr",
[junit4] 2> "127.0.0.1:10038_solr",
[junit4] 2> "127.0.0.1:10086_solr"]}}
[junit4] 2> 2154546 DEBUG (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.a.ComputePlanAction -- processing event: {
[junit4] 2> "id":"7c3dc1fda675Tbb8x7tcy164rfweb08jol5m0w",
[junit4] 2> "source":"node_lost_trigger",
[junit4] 2> "eventTime":136604689475189,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 136604689475189,
[junit4] 2> 136604668565089,
[junit4] 2> 136604671986439,
[junit4] 2> 136604684703439,
[junit4] 2> 136604675170889,
[junit4] 2> 136604692307289,
[junit4] 2> 136604678305839,
[junit4] 2> 136604681461739,
[junit4] 2> 136604695299239,
[junit4] 2> 136604698257039],
[junit4] 2> "_enqueue_time_":136609861116439,
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10085_solr",
[junit4] 2> "127.0.0.1:10018_solr",
[junit4] 2> "127.0.0.1:10049_solr",
[junit4] 2> "127.0.0.1:10087_solr",
[junit4] 2> "127.0.0.1:10074_solr",
[junit4] 2> "127.0.0.1:10046_solr",
[junit4] 2> "127.0.0.1:10113_solr",
[junit4] 2> "127.0.0.1:10035_solr",
[junit4] 2> "127.0.0.1:10038_solr",
[junit4] 2> "127.0.0.1:10086_solr"]}} with context properties: {BEFORE_ACTION=[compute]}
[junit4] 2> 2154560 INFO (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10106_solr&inPlaceMove=true&replica=core_node41
[junit4] 2> 2154571 INFO (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10094_solr&inPlaceMove=true&replica=core_node5
[junit4] 2> 2154582 INFO (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10091_solr&inPlaceMove=true&replica=core_node72
[junit4] 2> 2154591 INFO (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10034_solr&inPlaceMove=true&replica=core_node73
[junit4] 2> 2154598 INFO (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10031_solr&inPlaceMove=true&replica=core_node10
[junit4] 2> 2154605 INFO (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10097_solr&inPlaceMove=true&replica=core_node74
[junit4] 2> 2154610 INFO (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10078_solr&inPlaceMove=true&replica=core_node24
[junit4] 2> 2154614 INFO (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10075_solr&inPlaceMove=true&replica=core_node65
[junit4] 2> 2154617 INFO (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10050_solr&inPlaceMove=true&replica=core_node43
[junit4] 2> 2154619 INFO (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10024_solr&inPlaceMove=true&replica=core_node42
[junit4] 2> 2154619 INFO (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 136613539 sessionWrapper.createTime 136609921070689, this.sessionWrapper.createTime 136609921070689
[junit4] 2> 2154619 DEBUG (AutoscalingActionExecutor-8420-thread-1) [ ] o.a.s.c.a.ExecutePlanAction -- processing event: {
[junit4] 2> "id":"7c3dc1fda675Tbb8x7tcy164rfweb08jol5m0w",
[junit4] 2> "source":"node_lost_trigger",
[junit4] 2> "eventTime":136604689475189,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 136604689475189,
[junit4] 2> 136604668565089,
[junit4] 2> 136604671986439,
[junit4] 2> 136604684703439,
[junit4] 2> 136604675170889,
[junit4] 2> 136604692307289,
[junit4] 2> 136604678305839,
[junit4] 2> 136604681461739,
[junit4] 2> 136604695299239,
[junit4] 2> 136604698257039],
[junit4] 2> "_enqueue_time_":136609861116439,
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10085_solr",
[junit4] 2> "127.0.0.1:10018_solr",
[junit4] 2> "127.0.0.1:10049_solr",
[junit4] 2> "127.0.0.1:10087_solr",
[junit4] 2> "127.0.0.1:10074_solr",
[junit4] 2> "127.0.0.1:10046_solr",
[junit4] 2> "127.0.0.1:10113_solr",
[junit4] 2> "127.0.0.1:10035_solr",
[junit4] 2> "127.0.0.1:10038_solr",
[junit4] 2> "127.0.0.1:10086_solr"]}} with context properties: {operations=[org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@1b6a6404, org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@3ab062da, org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@102f997e, org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@2213cac8, org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@6a11c0ca, org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@538ed9e0, org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@3f865e86, org.apache.solr.client.so
[...truncated too long message...]
rter [org.apache.solr.metrics.reporters.SolrJmxReporter@2549944e: rootName = solr_32851, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@b3e55e
[junit4] 2> 203839 INFO (jetty-closer-202-thread-4) [ ] o.a.s.c.Overseer Overseer (id=99400986694778892-127.0.0.1:44601_solr-n_0000000000) closing
[junit4] 2> 203839 INFO (OverseerStateUpdate-99400986694778892-127.0.0.1:44601_solr-n_0000000000) [n:127.0.0.1:44601_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:44601_solr
[junit4] 2> 203839 WARN (OverseerAutoScalingTriggerThread-99400986694778892-127.0.0.1:44601_solr-n_0000000000) [n:127.0.0.1:44601_solr ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 203839 INFO (jetty-closer-202-thread-2) [ ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:32851_solr
[junit4] 2> 203840 INFO (jetty-closer-202-thread-2) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
[junit4] 2> 203840 INFO (zkCallback-231-thread-1-processing-n:127.0.0.1:44601_solr) [n:127.0.0.1:44601_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
[junit4] 2> 203840 INFO (zkCallback-233-thread-1-processing-n:127.0.0.1:34381_solr) [n:127.0.0.1:34381_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
[junit4] 2> 203840 INFO (zkCallback-229-thread-1-processing-n:127.0.0.1:32851_solr) [n:127.0.0.1:32851_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
[junit4] 2> 203840 INFO (zkCallback-1958-thread-1-processing-n:127.0.0.1:34381_solr x:collection1_shard1_replica_n1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:34381_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
[junit4] 2> 203841 INFO (coreCloseExecutor-729-thread-1) [n:127.0.0.1:34381_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.SolrCore [collection1_shard1_replica_n1] CLOSING SolrCore org.apache.solr.core.SolrCore@3e5ef7e8
[junit4] 2> 203842 INFO (zkCallback-233-thread-1-processing-n:127.0.0.1:34381_solr) [n:127.0.0.1:34381_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:34381_solr
[junit4] 2> 203842 INFO (jetty-closer-202-thread-4) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@f4f9736{/solr,null,UNAVAILABLE}
[junit4] 2> 203843 INFO (jetty-closer-202-thread-4) [ ] o.e.j.s.session Stopped scavenging
[junit4] 2> 203845 INFO (coreCloseExecutor-729-thread-1) [n:127.0.0.1:34381_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1.shard1.replica_n1, tag=1046411240
[junit4] 2> 203845 INFO (coreCloseExecutor-729-thread-1) [n:127.0.0.1:34381_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@3c1fb8e8: rootName = solr_34381, domain = solr.core.collection1.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.collection1.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@5fc54476
[junit4] 2> 203849 INFO (coreCloseExecutor-731-thread-1) [n:127.0.0.1:32851_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.SolrCore [collection1_shard2_replica_n2] CLOSING SolrCore org.apache.solr.core.SolrCore@30fe20c2
[junit4] 2> 203849 INFO (coreCloseExecutor-731-thread-1) [n:127.0.0.1:32851_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1.shard2.replica_n2, tag=821960898
[junit4] 2> 203849 INFO (coreCloseExecutor-731-thread-1) [n:127.0.0.1:32851_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@4a8598e6: rootName = solr_32851, domain = solr.core.collection1.shard2.replica_n2, service url = null, agent id = null] for registry solr.core.collection1.shard2.replica_n2 / com.codahale.metrics.MetricRegistry@39f70898
[junit4] 2> 203850 INFO (coreCloseExecutor-729-thread-1) [n:127.0.0.1:34381_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.collection1.shard1.leader, tag=1046411240
[junit4] 2> 203852 INFO (zkCallback-229-thread-1-processing-n:127.0.0.1:32851_solr) [n:127.0.0.1:32851_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:32851_solr
[junit4] 2> 203852 INFO (jetty-closer-202-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@23f35715{/solr,null,UNAVAILABLE}
[junit4] 2> 203853 INFO (jetty-closer-202-thread-1) [ ] o.e.j.s.session Stopped scavenging
[junit4] 2> 203856 INFO (coreCloseExecutor-731-thread-1) [n:127.0.0.1:32851_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.collection1.shard2.leader, tag=821960898
[junit4] 2> 203857 INFO (jetty-closer-202-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@1ec279a9{/solr,null,UNAVAILABLE}
[junit4] 2> 203857 INFO (jetty-closer-202-thread-2) [ ] o.e.j.s.session Stopped scavenging
[junit4] 2> 205335 WARN (zkCallback-235-thread-2-processing-n:127.0.0.1:37123_solr) [n:127.0.0.1:37123_solr ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 205335 INFO (jetty-closer-202-thread-3) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@1e0d12f{/solr,null,UNAVAILABLE}
[junit4] 2> 205335 INFO (jetty-closer-202-thread-3) [ ] o.e.j.s.session Stopped scavenging
[junit4] 2> 205336 ERROR (SUITE-StreamExpressionTest-seed#[6471768B25648F68]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 205336 INFO (SUITE-StreamExpressionTest-seed#[6471768B25648F68]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:42981 42981
[junit4] 2> 210104 INFO (Thread-144) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:42981 42981
[junit4] 2> 210104 WARN (Thread-144) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 100 /solr/aliases.json
[junit4] 2> 11 /solr/configs/conf
[junit4] 2> 7 /solr/configs/ml
[junit4] 2> 4 /solr/security.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 100 /solr/clusterprops.json
[junit4] 2> 100 /solr/clusterstate.json
[junit4] 2> 38 /solr/collections/destinationCollection/state.json
[junit4] 2> 27 /solr/collections/parallelDestinationCollection1/state.json
[junit4] 2> 20 /solr/collections/parallelDestinationCollection/state.json
[junit4] 2> 10 /solr/collections/mainCorpus1/state.json
[junit4] 2> 10 /solr/collections/destination/state.json
[junit4] 2> 10 /solr/collections/destination1/state.json
[junit4] 2> 10 /solr/collections/collection1/state.json
[junit4] 2> 10 /solr/collections/workQueue1/state.json
[junit4] 2> 10 /solr/collections/workQueue/state.json
[junit4] 2> 10 /solr/collections/uknownCollection/state.json
[junit4] 2> 10 /solr/collections/modelCollection/state.json
[junit4] 2> 10 /solr/collections/checkpointCollection/state.json
[junit4] 2> 10 /solr/collections/mainCorpus/state.json
[junit4] 2> 2 /solr/overseer_elect/election/99400986694778891-127.0.0.1:34381_solr-n_0000000001
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 100 /solr/live_nodes
[junit4] 2> 100 /solr/collections
[junit4] 2>
[junit4] 2> NOTE: leaving temporary files on disk at: /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-solrj/test/J2/temp/solr.client.solrj.io.stream.StreamExpressionTest_6471768B25648F68-001
[junit4] 2> NOTE: test params are: codec=HighCompressionCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=HIGH_COMPRESSION, chunkSize=6, maxDocsPerChunk=6, blockSize=855), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=HIGH_COMPRESSION, chunkSize=6, blockSize=855)), sim=RandomSimilarity(queryNorm=false): {}, locale=es-VE, timezone=Atlantic/Faeroe
[junit4] 2> NOTE: Linux 4.13.0-31-generic amd64/Oracle Corporation 10-ea (64-bit)/cpus=8,threads=1,free=140994872,total=525336576
[junit4] 2> NOTE: All tests run in this JVM: [TangentEvaluatorTest, SolrExampleXMLTest, HyperbolicSineEvaluatorTest, NamedListTest, UuidEvaluatorTest, TestCloudSolrClientConnections, CommonParamsTest, TestHash, CloudSolrClientBuilderTest, DocumentAnalysisResponseTest, LBHttpSolrClientBuilderTest, SineEvaluatorTest, SolrExampleJettyTest, ArrayEvaluatorTest, LessThanEqualToEvaluatorTest, OperationsTest, CloudSolrClientTest, TestCollectionAdminRequest, CoalesceEvaluatorTest, CosineEvaluatorTest, ExclusiveOrEvaluatorTest, SelectWithEvaluatorsTest, TestCoreAdmin, SolrExceptionTest, HttpSolrClientBuilderTest, JdbcDriverTest, StreamExpressionTest]
[junit4] Completed [146/146 (1!)] on J2 in 120.23s, 152 tests, 1 failure <<< FAILURES!
[...truncated 39995 lines...]
Re: [JENKINS-EA] Lucene-Solr-7.x-Linux (64bit/jdk-10-ea+37) - Build #
1229 - Still Unstable!
Posted by Joel Bernstein <jo...@gmail.com>.
Looks like the thresholds on this test need to be adjusted.
Joel Bernstein
http://joelsolr.blogspot.com/
On Tue, Jan 23, 2018 at 3:30 PM, Policeman Jenkins Server <
jenkins@thetaphi.de> wrote:
> Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Linux/1229/
> Java: 64bit/jdk-10-ea+37 -XX:+UseCompressedOops -XX:+UseParallelGC
>
> 2 tests failed.
> FAILED: org.apache.solr.client.solrj.io.stream.StreamExpressionTest.
> testFuzzyKmeans
>
> Error Message:
> expected:<1.3724537218127956E-4> but was:<0.0>
>
> Stack Trace:
> java.lang.AssertionError: expected:<1.3724537218127956E-4> but was:<0.0>
> at __randomizedtesting.SeedInfo.seed([6471768B25648F68:
> FEA595A4E052219]:0)
> at org.junit.Assert.fail(Assert.java:93)
> at org.junit.Assert.failNotEquals(Assert.java:647)
> at org.junit.Assert.assertEquals(Assert.java:443)
> at org.junit.Assert.assertEquals(Assert.java:512)
> at org.apache.solr.client.solrj.io.stream.StreamExpressionTest.
> testFuzzyKmeans(StreamExpressionTest.java:7140)
> at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native
> Method)
> at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(
> NativeMethodAccessorImpl.java:62)
> at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.
> invoke(DelegatingMethodAccessorImpl.java:43)
> at java.base/java.lang.reflect.Method.invoke(Method.java:564)
> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(
> RandomizedRunner.java:1737)
> at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(
> RandomizedRunner.java:934)
> at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(
> RandomizedRunner.java:970)
> at com.carrotsearch.randomizedtesting.
> RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
> at com.carrotsearch.randomizedtesting.rules.
> SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.
> java:57)
> at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(
> TestRuleSetupTeardownChained.java:49)
> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(
> AbstractBeforeAfterRule.java:45)
> at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(
> TestRuleThreadAndTestName.java:48)
> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures
> $1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
> at org.apache.lucene.util.TestRuleMarkFailure$1.
> evaluate(TestRuleMarkFailure.java:47)
> at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
> at com.carrotsearch.randomizedtesting.ThreadLeakControl$
> StatementRunner.run(ThreadLeakControl.java:368)
> at com.carrotsearch.randomizedtesting.ThreadLeakControl.
> forkTimeoutingTask(ThreadLeakControl.java:817)
> at com.carrotsearch.randomizedtesting.
> ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
> at com.carrotsearch.randomizedtesting.RandomizedRunner.
> runSingleTest(RandomizedRunner.java:943)
> at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(
> RandomizedRunner.java:829)
> at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(
> RandomizedRunner.java:879)
> at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(
> RandomizedRunner.java:890)
> at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
> at com.carrotsearch.randomizedtesting.rules.
> SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.
> java:57)
> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(
> AbstractBeforeAfterRule.java:45)
> at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(
> TestRuleStoreClassName.java:41)
> at com.carrotsearch.randomizedtesting.rules.
> NoShadowingOrOverridesOnMethodsRule$1.evaluate(
> NoShadowingOrOverridesOnMethodsRule.java:40)
> at com.carrotsearch.randomizedtesting.rules.
> NoShadowingOrOverridesOnMethodsRule$1.evaluate(
> NoShadowingOrOverridesOnMethodsRule.java:40)
> at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
> at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(
> TestRuleAssertionsRequired.java:53)
> at org.apache.lucene.util.TestRuleMarkFailure$1.
> evaluate(TestRuleMarkFailure.java:47)
> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures
> $1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(
> TestRuleIgnoreTestSuites.java:54)
> at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
> at com.carrotsearch.randomizedtesting.ThreadLeakControl$
> StatementRunner.run(ThreadLeakControl.java:368)
> at java.base/java.lang.Thread.run(Thread.java:844)
>
>
> FAILED: org.apache.solr.cloud.autoscaling.sim.
> TestLargeCluster.testNodeLost
>
> Error Message:
>
>
> Stack Trace:
> java.util.concurrent.TimeoutException
> at __randomizedtesting.SeedInfo.seed([8A7B7719C587386D:
> 356EB9E7466D5DEB]:0)
> at org.apache.solr.cloud.autoscaling.sim.SimSolrCloudTestCase.
> waitForState(SimSolrCloudTestCase.java:271)
> at org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.
> doTestNodeLost(TestLargeCluster.java:453)
> at org.apache.solr.cloud.autoscaling.sim.
> TestLargeCluster.testNodeLost(TestLargeCluster.java:295)
> at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native
> Method)
> at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(
> NativeMethodAccessorImpl.java:62)
> at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.
> invoke(DelegatingMethodAccessorImpl.java:43)
> at java.base/java.lang.reflect.Method.invoke(Method.java:564)
> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(
> RandomizedRunner.java:1737)
> at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(
> RandomizedRunner.java:934)
> at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(
> RandomizedRunner.java:970)
> at com.carrotsearch.randomizedtesting.
> RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
> at com.carrotsearch.randomizedtesting.rules.
> SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.
> java:57)
> at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(
> TestRuleSetupTeardownChained.java:49)
> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(
> AbstractBeforeAfterRule.java:45)
> at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(
> TestRuleThreadAndTestName.java:48)
> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures
> $1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
> at org.apache.lucene.util.TestRuleMarkFailure$1.
> evaluate(TestRuleMarkFailure.java:47)
> at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
> at com.carrotsearch.randomizedtesting.ThreadLeakControl$
> StatementRunner.run(ThreadLeakControl.java:368)
> at com.carrotsearch.randomizedtesting.ThreadLeakControl.
> forkTimeoutingTask(ThreadLeakControl.java:817)
> at com.carrotsearch.randomizedtesting.
> ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
> at com.carrotsearch.randomizedtesting.RandomizedRunner.
> runSingleTest(RandomizedRunner.java:943)
> at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(
> RandomizedRunner.java:829)
> at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(
> RandomizedRunner.java:879)
> at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(
> RandomizedRunner.java:890)
> at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
> at com.carrotsearch.randomizedtesting.rules.
> SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.
> java:57)
> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(
> AbstractBeforeAfterRule.java:45)
> at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(
> TestRuleStoreClassName.java:41)
> at com.carrotsearch.randomizedtesting.rules.
> NoShadowingOrOverridesOnMethodsRule$1.evaluate(
> NoShadowingOrOverridesOnMethodsRule.java:40)
> at com.carrotsearch.randomizedtesting.rules.
> NoShadowingOrOverridesOnMethodsRule$1.evaluate(
> NoShadowingOrOverridesOnMethodsRule.java:40)
> at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
> at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(
> TestRuleAssertionsRequired.java:53)
> at org.apache.lucene.util.TestRuleMarkFailure$1.
> evaluate(TestRuleMarkFailure.java:47)
> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures
> $1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(
> TestRuleIgnoreTestSuites.java:54)
> at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
> at com.carrotsearch.randomizedtesting.ThreadLeakControl$
> StatementRunner.run(ThreadLeakControl.java:368)
> at java.base/java.lang.Thread.run(Thread.java:844)
>
>
>
>
> Build Log:
> [...truncated 13657 lines...]
> [junit4] Suite: org.apache.solr.cloud.autoscaling.sim.TestLargeCluster
> [junit4] 2> 2151391 INFO (SUITE-TestLargeCluster-seed#[8A7B7719C587386D]-worker)
> [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null
> & java.security.egd=file:/dev/./urandom
> [junit4] 2> Creating dataDir: /home/jenkins/workspace/
> Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/
> solr.cloud.autoscaling.sim.TestLargeCluster_8A7B7719C587386D-001/init-
> core-data-001
> [junit4] 2> 2151392 INFO (SUITE-TestLargeCluster-seed#[8A7B7719C587386D]-worker)
> [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true)
> w/NUMERIC_DOCVALUES_SYSPROP=true
> [junit4] 2> 2151392 INFO (SUITE-TestLargeCluster-seed#[8A7B7719C587386D]-worker)
> [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true)
> via: @org.apache.solr.util.RandomizeSSL(reason="", ssl=0.0/0.0,
> value=0.0/0.0, clientAuth=0.0/0.0)
> [junit4] 2> 2151393 DEBUG (Simulated OverseerAutoScalingTriggerThread)
> [ ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
> [junit4] 2> 2151393 DEBUG (Simulated OverseerAutoScalingTriggerThread)
> [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with
> znode version 0
> [junit4] 2> 2151394 DEBUG (Simulated OverseerAutoScalingTriggerThread)
> [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10022_solr,
> 127.0.0.1:10018_solr, 127.0.0.1:10016_solr, 127.0.0.1:10021_solr,
> 127.0.0.1:10017_solr, 127.0.0.1:10020_solr, 127.0.0.1:10019_solr,
> 127.0.0.1:10023_solr]
> [junit4] 2> 2151394 DEBUG (Simulated OverseerAutoScalingTriggerThread)
> [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0,
> lastZnodeVersion -1
> [junit4] 2> 2151394 DEBUG (Simulated OverseerAutoScalingTriggerThread)
> [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto
> znodeVersion 0
> [junit4] 2> 2151394 DEBUG (Simulated OverseerAutoScalingTriggerThread)
> [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
> [junit4] 2> 2151394 DEBUG (Simulated OverseerAutoScalingTriggerThread)
> [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0,
> lastZnodeVersion 0
> [junit4] 2> 2151394 DEBUG (ScheduledTrigger-8416-thread-1) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 16
> [junit4] 2> 2151400 INFO (TEST-TestLargeCluster.testBasic-seed#[8A7B7719C587386D])
> [ ] o.a.s.SolrTestCaseJ4 ###Starting testBasic
> [junit4] 2> 2151400 DEBUG (TEST-TestLargeCluster.testBasic-seed#[8A7B7719C587386D])
> [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with
> znode version 1
> [junit4] 2> 2151400 DEBUG (Simulated OverseerAutoScalingTriggerThread)
> [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto
> znodeVersion 1
> [junit4] 2> 2151400 DEBUG (Simulated OverseerAutoScalingTriggerThread)
> [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
> [junit4] 2> 2151400 INFO (TEST-TestLargeCluster.testBasic-seed#[8A7B7719C587386D])
> [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and
> clearing object cache...
> [junit4] 2> 2151400 DEBUG (Simulated OverseerAutoScalingTriggerThread)
> [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
> [junit4] 2> 2151401 WARN (Simulated OverseerAutoScalingTriggerThread)
> [ ] o.a.s.c.a.OverseerTriggerThread Interrupted
> [junit4] 2> java.lang.InterruptedException
> [junit4] 2> at java.base/java.util.concurrent.locks.
> AbstractQueuedSynchronizer.acquireInterruptibly(
> AbstractQueuedSynchronizer.java:1259)
> [junit4] 2> at java.base/java.util.concurrent.locks.
> ReentrantLock.lockInterruptibly(ReentrantLock.java:317)
> [junit4] 2> at org.apache.solr.cloud.autoscaling.
> OverseerTriggerThread.run(OverseerTriggerThread.java:154)
> [junit4] 2> at java.base/java.lang.Thread.run(Thread.java:844)
> [junit4] 2> 2151401 DEBUG (TEST-TestLargeCluster.testBasic-seed#[8A7B7719C587386D])
> [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been
> closed explicitly
> [junit4] 2> 2151401 DEBUG (Simulated OverseerAutoScalingTriggerThread)
> [ ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
> [junit4] 2> 2151401 DEBUG (Simulated OverseerAutoScalingTriggerThread)
> [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with
> znode version 2
> [junit4] 2> 2151401 DEBUG (Simulated OverseerAutoScalingTriggerThread)
> [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10082_solr,
> 127.0.0.1:10018_solr, 127.0.0.1:10049_solr, 127.0.0.1:10104_solr,
> 127.0.0.1:10020_solr, 127.0.0.1:10065_solr, 127.0.0.1:10021_solr,
> 127.0.0.1:10066_solr, 127.0.0.1:10035_solr, 127.0.0.1:10051_solr,
> 127.0.0.1:10019_solr, 127.0.0.1:10096_solr, 127.0.0.1:10022_solr,
> 127.0.0.1:10033_solr, 127.0.0.1:10105_solr, 127.0.0.1:10017_solr,
> 127.0.0.1:10047_solr, 127.0.0.1:10102_solr, 127.0.0.1:10095_solr,
> 127.0.0.1:10081_solr, 127.0.0.1:10084_solr, 127.0.0.1:10036_solr,
> 127.0.0.1:10052_solr, 127.0.0.1:10079_solr, 127.0.0.1:10098_solr,
> 127.0.0.1:10054_solr, 127.0.0.1:10099_solr, 127.0.0.1:10101_solr,
> 127.0.0.1:10071_solr, 127.0.0.1:10023_solr, 127.0.0.1:10076_solr,
> 127.0.0.1:10107_solr, 127.0.0.1:10055_solr, 127.0.0.1:10016_solr,
> 127.0.0.1:10070_solr, 127.0.0.1:10077_solr, 127.0.0.1:10038_solr,
> 127.0.0.1:10032_solr, 127.0.0.1:10108_solr, 127.0.0.1:10093_solr,
> 127.0.0.1:10061_solr, 127.0.0.1:10067_solr, 127.0.0.1:10086_solr,
> 127.0.0.1:10089_solr, 127.0.0.1:10045_solr, 127.0.0.1:10042_solr,
> 127.0.0.1:10029_solr, 127.0.0.1:10083_solr, 127.0.0.1:10114_solr,
> 127.0.0.1:10080_solr, 127.0.0.1:10048_solr, 127.0.0.1:10064_solr,
> 127.0.0.1:10111_solr, 127.0.0.1:10043_solr, 127.0.0.1:10057_solr,
> 127.0.0.1:10112_solr, 127.0.0.1:10074_solr, 127.0.0.1:10026_solr,
> 127.0.0.1:10060_solr, 127.0.0.1:10073_solr, 127.0.0.1:10058_solr,
> 127.0.0.1:10113_solr, 127.0.0.1:10090_solr, 127.0.0.1:10088_solr,
> 127.0.0.1:10027_solr, 127.0.0.1:10039_solr, 127.0.0.1:10025_solr,
> 127.0.0.1:10044_solr, 127.0.0.1:10028_solr, 127.0.0.1:10092_solr,
> 127.0.0.1:10041_solr, 127.0.0.1:10087_solr, 127.0.0.1:10030_solr,
> 127.0.0.1:10040_solr, 127.0.0.1:10085_solr, 127.0.0.1:10046_solr,
> 127.0.0.1:10068_solr, 127.0.0.1:10063_solr, 127.0.0.1:10115_solr,
> 127.0.0.1:10069_solr, 127.0.0.1:10110_solr, 127.0.0.1:10062_solr,
> 127.0.0.1:10024_solr, 127.0.0.1:10050_solr, 127.0.0.1:10075_solr,
> 127.0.0.1:10078_solr, 127.0.0.1:10097_solr, 127.0.0.1:10031_solr,
> 127.0.0.1:10034_solr, 127.0.0.1:10091_solr, 127.0.0.1:10094_solr,
> 127.0.0.1:10106_solr, 127.0.0.1:10100_solr, 127.0.0.1:10037_solr,
> 127.0.0.1:10109_solr, 127.0.0.1:10053_solr, 127.0.0.1:10059_solr,
> 127.0.0.1:10103_solr, 127.0.0.1:10072_solr, 127.0.0.1:10056_solr]
> [junit4] 2> 2151401 DEBUG (Simulated OverseerAutoScalingTriggerThread)
> [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2,
> lastZnodeVersion -1
> [junit4] 2> 2151401 DEBUG (Simulated OverseerAutoScalingTriggerThread)
> [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto
> znodeVersion 2
> [junit4] 2> 2151401 DEBUG (Simulated OverseerAutoScalingTriggerThread)
> [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
> [junit4] 2> 2151401 DEBUG (Simulated OverseerAutoScalingTriggerThread)
> [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2,
> lastZnodeVersion 2
> [junit4] 2> 2151402 DEBUG (ScheduledTrigger-8419-thread-1) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 100
> [junit4] 2> 2151422 DEBUG (ScheduledTrigger-8419-thread-1) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 100
> [junit4] 2> 2151442 DEBUG (ScheduledTrigger-8419-thread-2) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 100
> [junit4] 2> 2151463 DEBUG (ScheduledTrigger-8419-thread-1) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 100
> [junit4] 2> 2151483 DEBUG (ScheduledTrigger-8419-thread-1) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 100
> [junit4] 2> 2151503 DEBUG (simCloudManagerPool-8418-thread-1) [ ]
> o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
> [junit4] 2> 2151503 DEBUG (simCloudManagerPool-8418-thread-1) [ ]
> o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode
> version 3
> [junit4] 2> 2151503 DEBUG (simCloudManagerPool-8418-thread-1) [ ]
> o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10082_solr,
> 127.0.0.1:10018_solr, 127.0.0.1:10049_solr, 127.0.0.1:10104_solr,
> 127.0.0.1:10020_solr, 127.0.0.1:10065_solr, 127.0.0.1:10021_solr,
> 127.0.0.1:10066_solr, 127.0.0.1:10035_solr, 127.0.0.1:10051_solr,
> 127.0.0.1:10019_solr, 127.0.0.1:10096_solr, 127.0.0.1:10022_solr,
> 127.0.0.1:10033_solr, 127.0.0.1:10105_solr, 127.0.0.1:10017_solr,
> 127.0.0.1:10047_solr, 127.0.0.1:10102_solr, 127.0.0.1:10095_solr,
> 127.0.0.1:10081_solr, 127.0.0.1:10084_solr, 127.0.0.1:10036_solr,
> 127.0.0.1:10052_solr, 127.0.0.1:10079_solr, 127.0.0.1:10098_solr,
> 127.0.0.1:10054_solr, 127.0.0.1:10099_solr, 127.0.0.1:10101_solr,
> 127.0.0.1:10071_solr, 127.0.0.1:10023_solr, 127.0.0.1:10076_solr,
> 127.0.0.1:10107_solr, 127.0.0.1:10055_solr, 127.0.0.1:10016_solr,
> 127.0.0.1:10070_solr, 127.0.0.1:10077_solr, 127.0.0.1:10038_solr,
> 127.0.0.1:10032_solr, 127.0.0.1:10108_solr, 127.0.0.1:10093_solr,
> 127.0.0.1:10061_solr, 127.0.0.1:10067_solr, 127.0.0.1:10086_solr,
> 127.0.0.1:10089_solr, 127.0.0.1:10045_solr, 127.0.0.1:10042_solr,
> 127.0.0.1:10029_solr, 127.0.0.1:10083_solr, 127.0.0.1:10114_solr,
> 127.0.0.1:10080_solr, 127.0.0.1:10048_solr, 127.0.0.1:10064_solr,
> 127.0.0.1:10111_solr, 127.0.0.1:10043_solr, 127.0.0.1:10057_solr,
> 127.0.0.1:10112_solr, 127.0.0.1:10074_solr, 127.0.0.1:10026_solr,
> 127.0.0.1:10060_solr, 127.0.0.1:10073_solr, 127.0.0.1:10058_solr,
> 127.0.0.1:10113_solr, 127.0.0.1:10090_solr, 127.0.0.1:10088_solr,
> 127.0.0.1:10027_solr, 127.0.0.1:10039_solr, 127.0.0.1:10025_solr,
> 127.0.0.1:10044_solr, 127.0.0.1:10028_solr, 127.0.0.1:10092_solr,
> 127.0.0.1:10041_solr, 127.0.0.1:10087_solr, 127.0.0.1:10030_solr,
> 127.0.0.1:10040_solr, 127.0.0.1:10085_solr, 127.0.0.1:10046_solr,
> 127.0.0.1:10068_solr, 127.0.0.1:10063_solr, 127.0.0.1:10115_solr,
> 127.0.0.1:10069_solr, 127.0.0.1:10110_solr, 127.0.0.1:10062_solr,
> 127.0.0.1:10024_solr, 127.0.0.1:10050_solr, 127.0.0.1:10075_solr,
> 127.0.0.1:10078_solr, 127.0.0.1:10097_solr, 127.0.0.1:10031_solr,
> 127.0.0.1:10034_solr, 127.0.0.1:10091_solr, 127.0.0.1:10094_solr,
> 127.0.0.1:10106_solr, 127.0.0.1:10100_solr, 127.0.0.1:10037_solr,
> 127.0.0.1:10109_solr, 127.0.0.1:10053_solr, 127.0.0.1:10059_solr,
> 127.0.0.1:10103_solr, 127.0.0.1:10072_solr, 127.0.0.1:10056_solr]
> [junit4] 2> 2151503 DEBUG (ScheduledTrigger-8419-thread-1) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 100
> [junit4] 2> 2151503 DEBUG (simCloudManagerPool-8418-thread-1) [ ]
> o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10082_solr,
> 127.0.0.1:10018_solr, 127.0.0.1:10049_solr, 127.0.0.1:10104_solr,
> 127.0.0.1:10020_solr, 127.0.0.1:10065_solr, 127.0.0.1:10021_solr,
> 127.0.0.1:10066_solr, 127.0.0.1:10035_solr, 127.0.0.1:10051_solr,
> 127.0.0.1:10019_solr, 127.0.0.1:10096_solr, 127.0.0.1:10022_solr,
> 127.0.0.1:10033_solr, 127.0.0.1:10105_solr, 127.0.0.1:10017_solr,
> 127.0.0.1:10047_solr, 127.0.0.1:10102_solr, 127.0.0.1:10095_solr,
> 127.0.0.1:10081_solr, 127.0.0.1:10084_solr, 127.0.0.1:10036_solr,
> 127.0.0.1:10052_solr, 127.0.0.1:10079_solr, 127.0.0.1:10098_solr,
> 127.0.0.1:10054_solr, 127.0.0.1:10099_solr, 127.0.0.1:10101_solr,
> 127.0.0.1:10071_solr, 127.0.0.1:10023_solr, 127.0.0.1:10076_solr,
> 127.0.0.1:10107_solr, 127.0.0.1:10055_solr, 127.0.0.1:10016_solr,
> 127.0.0.1:10070_solr, 127.0.0.1:10077_solr, 127.0.0.1:10038_solr,
> 127.0.0.1:10032_solr, 127.0.0.1:10108_solr, 127.0.0.1:10093_solr,
> 127.0.0.1:10061_solr, 127.0.0.1:10067_solr, 127.0.0.1:10086_solr,
> 127.0.0.1:10089_solr, 127.0.0.1:10045_solr, 127.0.0.1:10042_solr,
> 127.0.0.1:10029_solr, 127.0.0.1:10083_solr, 127.0.0.1:10114_solr,
> 127.0.0.1:10080_solr, 127.0.0.1:10048_solr, 127.0.0.1:10064_solr,
> 127.0.0.1:10111_solr, 127.0.0.1:10043_solr, 127.0.0.1:10057_solr,
> 127.0.0.1:10112_solr, 127.0.0.1:10074_solr, 127.0.0.1:10026_solr,
> 127.0.0.1:10060_solr, 127.0.0.1:10073_solr, 127.0.0.1:10058_solr,
> 127.0.0.1:10113_solr, 127.0.0.1:10090_solr, 127.0.0.1:10088_solr,
> 127.0.0.1:10027_solr, 127.0.0.1:10039_solr, 127.0.0.1:10025_solr,
> 127.0.0.1:10044_solr, 127.0.0.1:10028_solr, 127.0.0.1:10092_solr,
> 127.0.0.1:10041_solr, 127.0.0.1:10087_solr, 127.0.0.1:10030_solr,
> 127.0.0.1:10040_solr, 127.0.0.1:10085_solr, 127.0.0.1:10046_solr,
> 127.0.0.1:10068_solr, 127.0.0.1:10063_solr, 127.0.0.1:10115_solr,
> 127.0.0.1:10069_solr, 127.0.0.1:10110_solr, 127.0.0.1:10062_solr,
> 127.0.0.1:10024_solr, 127.0.0.1:10050_solr, 127.0.0.1:10075_solr,
> 127.0.0.1:10078_solr, 127.0.0.1:10097_solr, 127.0.0.1:10031_solr,
> 127.0.0.1:10034_solr, 127.0.0.1:10091_solr, 127.0.0.1:10094_solr,
> 127.0.0.1:10106_solr, 127.0.0.1:10100_solr, 127.0.0.1:10037_solr,
> 127.0.0.1:10109_solr, 127.0.0.1:10053_solr, 127.0.0.1:10059_solr,
> 127.0.0.1:10103_solr, 127.0.0.1:10072_solr, 127.0.0.1:10056_solr]
> [junit4] 2> 2151503 DEBUG (Simulated OverseerAutoScalingTriggerThread)
> [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto
> znodeVersion 3
> [junit4] 2> 2151504 DEBUG (Simulated OverseerAutoScalingTriggerThread)
> [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
> [junit4] 2> 2151504 DEBUG (ScheduledTrigger-8419-thread-3) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 100
> [junit4] 2> 2151504 DEBUG (Simulated OverseerAutoScalingTriggerThread)
> [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3,
> lastZnodeVersion 3
> [junit4] 2> 2151505 DEBUG (simCloudManagerPool-8418-thread-2) [ ]
> o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
> [junit4] 2> 2151505 DEBUG (simCloudManagerPool-8418-thread-2) [ ]
> o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode
> version 4
> [junit4] 2> 2151505 DEBUG (simCloudManagerPool-8418-thread-2) [ ]
> o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10082_solr,
> 127.0.0.1:10018_solr, 127.0.0.1:10049_solr, 127.0.0.1:10104_solr,
> 127.0.0.1:10020_solr, 127.0.0.1:10065_solr, 127.0.0.1:10021_solr,
> 127.0.0.1:10066_solr, 127.0.0.1:10035_solr, 127.0.0.1:10051_solr,
> 127.0.0.1:10019_solr, 127.0.0.1:10096_solr, 127.0.0.1:10022_solr,
> 127.0.0.1:10033_solr, 127.0.0.1:10105_solr, 127.0.0.1:10017_solr,
> 127.0.0.1:10047_solr, 127.0.0.1:10102_solr, 127.0.0.1:10095_solr,
> 127.0.0.1:10081_solr, 127.0.0.1:10084_solr, 127.0.0.1:10036_solr,
> 127.0.0.1:10052_solr, 127.0.0.1:10079_solr, 127.0.0.1:10098_solr,
> 127.0.0.1:10054_solr, 127.0.0.1:10099_solr, 127.0.0.1:10101_solr,
> 127.0.0.1:10071_solr, 127.0.0.1:10023_solr, 127.0.0.1:10076_solr,
> 127.0.0.1:10107_solr, 127.0.0.1:10055_solr, 127.0.0.1:10016_solr,
> 127.0.0.1:10070_solr, 127.0.0.1:10077_solr, 127.0.0.1:10038_solr,
> 127.0.0.1:10032_solr, 127.0.0.1:10108_solr, 127.0.0.1:10093_solr,
> 127.0.0.1:10061_solr, 127.0.0.1:10067_solr, 127.0.0.1:10086_solr,
> 127.0.0.1:10089_solr, 127.0.0.1:10045_solr, 127.0.0.1:10042_solr,
> 127.0.0.1:10029_solr, 127.0.0.1:10083_solr, 127.0.0.1:10114_solr,
> 127.0.0.1:10080_solr, 127.0.0.1:10048_solr, 127.0.0.1:10064_solr,
> 127.0.0.1:10111_solr, 127.0.0.1:10043_solr, 127.0.0.1:10057_solr,
> 127.0.0.1:10112_solr, 127.0.0.1:10074_solr, 127.0.0.1:10026_solr,
> 127.0.0.1:10060_solr, 127.0.0.1:10073_solr, 127.0.0.1:10058_solr,
> 127.0.0.1:10113_solr, 127.0.0.1:10090_solr, 127.0.0.1:10088_solr,
> 127.0.0.1:10027_solr, 127.0.0.1:10039_solr, 127.0.0.1:10025_solr,
> 127.0.0.1:10044_solr, 127.0.0.1:10028_solr, 127.0.0.1:10092_solr,
> 127.0.0.1:10041_solr, 127.0.0.1:10087_solr, 127.0.0.1:10030_solr,
> 127.0.0.1:10040_solr, 127.0.0.1:10085_solr, 127.0.0.1:10046_solr,
> 127.0.0.1:10068_solr, 127.0.0.1:10063_solr, 127.0.0.1:10115_solr,
> 127.0.0.1:10069_solr, 127.0.0.1:10110_solr, 127.0.0.1:10062_solr,
> 127.0.0.1:10024_solr, 127.0.0.1:10050_solr, 127.0.0.1:10075_solr,
> 127.0.0.1:10078_solr, 127.0.0.1:10097_solr, 127.0.0.1:10031_solr,
> 127.0.0.1:10034_solr, 127.0.0.1:10091_solr, 127.0.0.1:10094_solr,
> 127.0.0.1:10106_solr, 127.0.0.1:10100_solr, 127.0.0.1:10037_solr,
> 127.0.0.1:10109_solr, 127.0.0.1:10053_solr, 127.0.0.1:10059_solr,
> 127.0.0.1:10103_solr, 127.0.0.1:10072_solr, 127.0.0.1:10056_solr]
> [junit4] 2> 2151505 DEBUG (simCloudManagerPool-8418-thread-2) [ ]
> o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10082_solr,
> 127.0.0.1:10018_solr, 127.0.0.1:10049_solr, 127.0.0.1:10104_solr,
> 127.0.0.1:10020_solr, 127.0.0.1:10065_solr, 127.0.0.1:10021_solr,
> 127.0.0.1:10066_solr, 127.0.0.1:10035_solr, 127.0.0.1:10051_solr,
> 127.0.0.1:10019_solr, 127.0.0.1:10096_solr, 127.0.0.1:10022_solr,
> 127.0.0.1:10033_solr, 127.0.0.1:10105_solr, 127.0.0.1:10017_solr,
> 127.0.0.1:10047_solr, 127.0.0.1:10102_solr, 127.0.0.1:10095_solr,
> 127.0.0.1:10081_solr, 127.0.0.1:10084_solr, 127.0.0.1:10036_solr,
> 127.0.0.1:10052_solr, 127.0.0.1:10079_solr, 127.0.0.1:10098_solr,
> 127.0.0.1:10054_solr, 127.0.0.1:10099_solr, 127.0.0.1:10101_solr,
> 127.0.0.1:10071_solr, 127.0.0.1:10023_solr, 127.0.0.1:10076_solr,
> 127.0.0.1:10107_solr, 127.0.0.1:10055_solr, 127.0.0.1:10016_solr,
> 127.0.0.1:10070_solr, 127.0.0.1:10077_solr, 127.0.0.1:10038_solr,
> 127.0.0.1:10032_solr, 127.0.0.1:10108_solr, 127.0.0.1:10093_solr,
> 127.0.0.1:10061_solr, 127.0.0.1:10067_solr, 127.0.0.1:10086_solr,
> 127.0.0.1:10089_solr, 127.0.0.1:10045_solr, 127.0.0.1:10042_solr,
> 127.0.0.1:10029_solr, 127.0.0.1:10083_solr, 127.0.0.1:10114_solr,
> 127.0.0.1:10080_solr, 127.0.0.1:10048_solr, 127.0.0.1:10064_solr,
> 127.0.0.1:10111_solr, 127.0.0.1:10043_solr, 127.0.0.1:10057_solr,
> 127.0.0.1:10112_solr, 127.0.0.1:10074_solr, 127.0.0.1:10026_solr,
> 127.0.0.1:10060_solr, 127.0.0.1:10073_solr, 127.0.0.1:10058_solr,
> 127.0.0.1:10113_solr, 127.0.0.1:10090_solr, 127.0.0.1:10088_solr,
> 127.0.0.1:10027_solr, 127.0.0.1:10039_solr, 127.0.0.1:10025_solr,
> 127.0.0.1:10044_solr, 127.0.0.1:10028_solr, 127.0.0.1:10092_solr,
> 127.0.0.1:10041_solr, 127.0.0.1:10087_solr, 127.0.0.1:10030_solr,
> 127.0.0.1:10040_solr, 127.0.0.1:10085_solr, 127.0.0.1:10046_solr,
> 127.0.0.1:10068_solr, 127.0.0.1:10063_solr, 127.0.0.1:10115_solr,
> 127.0.0.1:10069_solr, 127.0.0.1:10110_solr, 127.0.0.1:10062_solr,
> 127.0.0.1:10024_solr, 127.0.0.1:10050_solr, 127.0.0.1:10075_solr,
> 127.0.0.1:10078_solr, 127.0.0.1:10097_solr, 127.0.0.1:10031_solr,
> 127.0.0.1:10034_solr, 127.0.0.1:10091_solr, 127.0.0.1:10094_solr,
> 127.0.0.1:10106_solr, 127.0.0.1:10100_solr, 127.0.0.1:10037_solr,
> 127.0.0.1:10109_solr, 127.0.0.1:10053_solr, 127.0.0.1:10059_solr,
> 127.0.0.1:10103_solr, 127.0.0.1:10072_solr, 127.0.0.1:10056_solr]
> [junit4] 2> 2151506 DEBUG (Simulated OverseerAutoScalingTriggerThread)
> [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto
> znodeVersion 4
> [junit4] 2> 2151506 DEBUG (Simulated OverseerAutoScalingTriggerThread)
> [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
> [junit4] 2> 2151506 DEBUG (Simulated OverseerAutoScalingTriggerThread)
> [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4,
> lastZnodeVersion 4
> [junit4] 2> 2151524 DEBUG (ScheduledTrigger-8419-thread-1) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 100
> [junit4] 2> 2151524 DEBUG (ScheduledTrigger-8419-thread-3) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 100
> [junit4] 2> 2151544 DEBUG (ScheduledTrigger-8419-thread-4) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 100
> [junit4] 2> 2151544 DEBUG (ScheduledTrigger-8419-thread-4) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 100
> [junit4] 2> 2151564 DEBUG (ScheduledTrigger-8419-thread-2) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 100
> [junit4] 2> 2151564 DEBUG (ScheduledTrigger-8419-thread-1) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 100
> [junit4] 2> 2151585 DEBUG (ScheduledTrigger-8419-thread-4) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 100
> [junit4] 2> 2151585 DEBUG (ScheduledTrigger-8419-thread-2) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 100
> [junit4] 2> 2151605 DEBUG (ScheduledTrigger-8419-thread-1) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 100
> [junit4] 2> 2151605 DEBUG (ScheduledTrigger-8419-thread-3) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 100
> [junit4] 2> 2151625 DEBUG (ScheduledTrigger-8419-thread-4) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 100
> [junit4] 2> 2151625 DEBUG (ScheduledTrigger-8419-thread-2) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 100
> [junit4] 2> 2151645 DEBUG (ScheduledTrigger-8419-thread-4) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 100
> [junit4] 2> 2151645 DEBUG (ScheduledTrigger-8419-thread-3) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 100
> [junit4] 2> 2151666 DEBUG (ScheduledTrigger-8419-thread-4) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 100
> [junit4] 2> 2151666 DEBUG (ScheduledTrigger-8419-thread-2) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 100
> [junit4] 2> 2151686 DEBUG (ScheduledTrigger-8419-thread-3) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 100
> [junit4] 2> 2151686 DEBUG (ScheduledTrigger-8419-thread-4) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 100
> [junit4] 2> 2151706 DEBUG (ScheduledTrigger-8419-thread-2) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 100
> [junit4] 2> 2151706 DEBUG (ScheduledTrigger-8419-thread-3) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 100
> [junit4] 2> 2151726 DEBUG (ScheduledTrigger-8419-thread-4) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 100
> [junit4] 2> 2151726 DEBUG (ScheduledTrigger-8419-thread-2) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 100
> [junit4] 2> 2151747 DEBUG (ScheduledTrigger-8419-thread-3) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 100
> [junit4] 2> 2151747 DEBUG (ScheduledTrigger-8419-thread-4) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 100
> [junit4] 2> 2151767 DEBUG (ScheduledTrigger-8419-thread-3) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 100
> [junit4] 2> 2151767 DEBUG (ScheduledTrigger-8419-thread-2) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 100
> [junit4] 2> 2151787 DEBUG (ScheduledTrigger-8419-thread-3) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 100
> [junit4] 2> 2151787 DEBUG (ScheduledTrigger-8419-thread-4) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 100
> [junit4] 2> 2151807 DEBUG (ScheduledTrigger-8419-thread-3) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 100
> [junit4] 2> 2151807 DEBUG (ScheduledTrigger-8419-thread-2) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 100
> [junit4] 2> 2151828 DEBUG (ScheduledTrigger-8419-thread-3) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 100
> [junit4] 2> 2151828 DEBUG (ScheduledTrigger-8419-thread-4) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 100
> [junit4] 2> 2151848 DEBUG (ScheduledTrigger-8419-thread-3) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 100
> [junit4] 2> 2151848 DEBUG (ScheduledTrigger-8419-thread-2) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 100
> [junit4] 2> 2151868 DEBUG (ScheduledTrigger-8419-thread-3) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 100
> [junit4] 2> 2151868 DEBUG (ScheduledTrigger-8419-thread-4) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 100
> [junit4] 2> 2151889 DEBUG (ScheduledTrigger-8419-thread-3) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 100
> [junit4] 2> 2151889 DEBUG (ScheduledTrigger-8419-thread-2) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 100
> [junit4] 2> 2151909 DEBUG (ScheduledTrigger-8419-thread-4) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 100
> [junit4] 2> 2151909 DEBUG (ScheduledTrigger-8419-thread-3) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 100
> [junit4] 2> 2151929 DEBUG (ScheduledTrigger-8419-thread-2) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 100
> [junit4] 2> 2151929 DEBUG (ScheduledTrigger-8419-thread-4) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 100
> [junit4] 2> 2151949 DEBUG (ScheduledTrigger-8419-thread-3) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 100
> [junit4] 2> 2151949 DEBUG (ScheduledTrigger-8419-thread-2) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 100
> [junit4] 2> 2151970 DEBUG (ScheduledTrigger-8419-thread-3) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 100
> [junit4] 2> 2151970 DEBUG (ScheduledTrigger-8419-thread-4) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 100
> [junit4] 2> 2151990 DEBUG (ScheduledTrigger-8419-thread-3) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 100
> [junit4] 2> 2151990 DEBUG (ScheduledTrigger-8419-thread-2) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 100
> [junit4] 2> 2152010 DEBUG (ScheduledTrigger-8419-thread-3) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 100
> [junit4] 2> 2152010 DEBUG (ScheduledTrigger-8419-thread-4) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 100
> [junit4] 2> 2152031 DEBUG (ScheduledTrigger-8419-thread-3) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 100
> [junit4] 2> 2152031 DEBUG (ScheduledTrigger-8419-thread-2) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 100
> [junit4] 2> 2152051 DEBUG (ScheduledTrigger-8419-thread-4) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 100
> [junit4] 2> 2152051 DEBUG (ScheduledTrigger-8419-thread-3) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 100
> [junit4] 2> 2152071 DEBUG (ScheduledTrigger-8419-thread-2) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 100
> [junit4] 2> 2152071 DEBUG (ScheduledTrigger-8419-thread-4) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 100
> [junit4] 2> 2152091 DEBUG (ScheduledTrigger-8419-thread-3) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 100
> [junit4] 2> 2152091 DEBUG (ScheduledTrigger-8419-thread-2) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 100
> [junit4] 2> 2152112 DEBUG (ScheduledTrigger-8419-thread-4) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 100
> [junit4] 2> 2152112 DEBUG (ScheduledTrigger-8419-thread-1) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 100
> [junit4] 2> 2152132 DEBUG (ScheduledTrigger-8419-thread-4) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 100
> [junit4] 2> 2152132 DEBUG (ScheduledTrigger-8419-thread-2) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 100
> [junit4] 2> 2152152 DEBUG (ScheduledTrigger-8419-thread-1) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 100
> [junit4] 2> 2152152 DEBUG (ScheduledTrigger-8419-thread-4) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 100
> [junit4] 2> 2152172 DEBUG (ScheduledTrigger-8419-thread-1) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 100
> [junit4] 2> 2152172 DEBUG (ScheduledTrigger-8419-thread-2) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 100
> [junit4] 2> 2152193 DEBUG (ScheduledTrigger-8419-thread-1) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 100
> [junit4] 2> 2152193 DEBUG (ScheduledTrigger-8419-thread-4) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 100
> [junit4] 2> 2152213 DEBUG (ScheduledTrigger-8419-thread-1) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 100
> [junit4] 2> 2152213 DEBUG (ScheduledTrigger-8419-thread-2) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 100
> [junit4] 2> 2152233 DEBUG (ScheduledTrigger-8419-thread-1) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 100
> [junit4] 2> 2152233 DEBUG (ScheduledTrigger-8419-thread-4) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 100
> [junit4] 2> 2152253 DEBUG (ScheduledTrigger-8419-thread-1) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 100
> [junit4] 2> 2152253 DEBUG (ScheduledTrigger-8419-thread-2) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 100
> [junit4] 2> 2152274 DEBUG (ScheduledTrigger-8419-thread-4) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 100
> [junit4] 2> 2152274 DEBUG (ScheduledTrigger-8419-thread-1) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 100
> [junit4] 2> 2152294 DEBUG (ScheduledTrigger-8419-thread-3) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 100
> [junit4] 2> 2152294 DEBUG (ScheduledTrigger-8419-thread-2) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 100
> [junit4] 2> 2152314 DEBUG (ScheduledTrigger-8419-thread-3) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 100
> [junit4] 2> 2152314 DEBUG (ScheduledTrigger-8419-thread-1) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 100
> [junit4] 2> 2152335 DEBUG (ScheduledTrigger-8419-thread-3) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 100
> [junit4] 2> 2152335 DEBUG (ScheduledTrigger-8419-thread-2) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 100
> [junit4] 2> 2152355 DEBUG (ScheduledTrigger-8419-thread-1) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 100
> [junit4] 2> 2152355 DEBUG (ScheduledTrigger-8419-thread-3) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 100
> [junit4] 2> 2152375 DEBUG (ScheduledTrigger-8419-thread-1) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 100
> [junit4] 2> 2152375 DEBUG (ScheduledTrigger-8419-thread-2) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 100
> [junit4] 2> 2152377 INFO (TEST-TestLargeCluster.testBasic-seed#[8A7B7719C587386D])
> [ ] o.a.s.c.a.s.TestLargeCluster Ready after 37704ms
> [junit4] 2> 2152381 DEBUG (simCloudManagerPool-8418-thread-79) [
> ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testBasic
> / shard2: {"core_node18":{
> [junit4] 2> "node_name":"127.0.0.1:10023_solr",
> [junit4] 2> "core":"testBasic_shard2_replica_n18",
> [junit4] 2> "leader":"true",
> [junit4] 2> "state":"active",
> [junit4] 2> "type":"NRT",
> [junit4] 2> "INDEX.sizeInBytes":123450000,
> [junit4] 2> "shard":"shard2",
> [junit4] 2> "collection":"testBasic"}}
> [junit4] 2> 2152381 DEBUG (simCloudManagerPool-8418-thread-80) [
> ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testBasic
> / shard3: {"core_node39":{
> [junit4] 2> "node_name":"127.0.0.1:10104_solr",
> [junit4] 2> "core":"testBasic_shard3_replica_t39",
> [junit4] 2> "leader":"true",
> [junit4] 2> "state":"active",
> [junit4] 2> "type":"TLOG",
> [junit4] 2> "INDEX.sizeInBytes":123450000,
> [junit4] 2> "shard":"shard3",
> [junit4] 2> "collection":"testBasic"}}
> [junit4] 2> 2152381 DEBUG (simCloudManagerPool-8418-thread-83) [
> ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testBasic
> / shard1: {"core_node1":{
> [junit4] 2> "node_name":"127.0.0.1:10101_solr",
> [junit4] 2> "core":"testBasic_shard1_replica_n1",
> [junit4] 2> "leader":"true",
> [junit4] 2> "state":"active",
> [junit4] 2> "type":"NRT",
> [junit4] 2> "INDEX.sizeInBytes":123450000,
> [junit4] 2> "shard":"shard1",
> [junit4] 2> "collection":"testBasic"}}
> [junit4] 2> 2152381 DEBUG (simCloudManagerPool-8418-thread-81) [
> ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testBasic
> / shard4: {"core_node54":{
> [junit4] 2> "node_name":"127.0.0.1:10064_solr",
> [junit4] 2> "core":"testBasic_shard4_replica_t54",
> [junit4] 2> "leader":"true",
> [junit4] 2> "state":"active",
> [junit4] 2> "type":"TLOG",
> [junit4] 2> "INDEX.sizeInBytes":123450000,
> [junit4] 2> "shard":"shard4",
> [junit4] 2> "collection":"testBasic"}}
> [junit4] 2> 2152382 DEBUG (simCloudManagerPool-8418-thread-82) [
> ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testBasic
> / shard5: {"core_node65":{
> [junit4] 2> "node_name":"127.0.0.1:10035_solr",
> [junit4] 2> "core":"testBasic_shard5_replica_n65",
> [junit4] 2> "leader":"true",
> [junit4] 2> "state":"active",
> [junit4] 2> "type":"NRT",
> [junit4] 2> "INDEX.sizeInBytes":123450000,
> [junit4] 2> "shard":"shard5",
> [junit4] 2> "collection":"testBasic"}}
> [junit4] 2> 2152395 DEBUG (ScheduledTrigger-8419-thread-1) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 96
> [junit4] 2> 2152395 DEBUG (ScheduledTrigger-8419-thread-3) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 96
> [junit4] 2> 2152396 DEBUG (ScheduledTrigger-8419-thread-1) [ ]
> o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10019_solr
> [junit4] 2> 2152396 DEBUG (ScheduledTrigger-8419-thread-3) [ ]
> o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10019_solr
> [junit4] 2> 2152396 DEBUG (ScheduledTrigger-8419-thread-1) [ ]
> o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10105_solr
> [junit4] 2> 2152396 DEBUG (ScheduledTrigger-8419-thread-3) [ ]
> o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10105_solr
> [junit4] 2> 2152396 DEBUG (ScheduledTrigger-8419-thread-1) [ ]
> o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10052_solr
> [junit4] 2> 2152396 DEBUG (ScheduledTrigger-8419-thread-1) [ ]
> o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10026_solr
> [junit4] 2> 2152396 DEBUG (ScheduledTrigger-8419-thread-3) [ ]
> o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10052_solr
> [junit4] 2> 2152396 DEBUG (ScheduledTrigger-8419-thread-3) [ ]
> o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10026_solr
> [junit4] 2> 2152416 DEBUG (ScheduledTrigger-8419-thread-1) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 92
> [junit4] 2> 2152416 DEBUG (ScheduledTrigger-8419-thread-1) [ ]
> o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10022_solr
> [junit4] 2> 2152416 DEBUG (ScheduledTrigger-8419-thread-1) [ ]
> o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10036_solr
> [junit4] 2> 2152416 DEBUG (ScheduledTrigger-8419-thread-1) [ ]
> o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10071_solr
> [junit4] 2> 2152416 DEBUG (ScheduledTrigger-8419-thread-1) [ ]
> o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10076_solr
> [junit4] 2> 2152416 DEBUG (ScheduledTrigger-8419-thread-1) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 92
> [junit4] 2> 2152416 DEBUG (ScheduledTrigger-8419-thread-1) [ ]
> o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10022_solr
> [junit4] 2> 2152416 DEBUG (ScheduledTrigger-8419-thread-1) [ ]
> o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10036_solr
> [junit4] 2> 2152416 DEBUG (ScheduledTrigger-8419-thread-1) [ ]
> o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10071_solr
> [junit4] 2> 2152416 DEBUG (ScheduledTrigger-8419-thread-1) [ ]
> o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10076_solr
> [junit4] 2> 2152436 DEBUG (ScheduledTrigger-8419-thread-1) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 92
> [junit4] 2> 2152437 DEBUG (ScheduledTrigger-8419-thread-3) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 92
> [junit4] 2> 2152457 DEBUG (ScheduledTrigger-8419-thread-2) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 92
> [junit4] 2> 2152457 DEBUG (ScheduledTrigger-8419-thread-2) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 92
> [junit4] 2> 2152477 DEBUG (ScheduledTrigger-8419-thread-4) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 92
> [junit4] 2> 2152477 DEBUG (ScheduledTrigger-8419-thread-1) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 92
> [junit4] 2> 2152497 DEBUG (ScheduledTrigger-8419-thread-4) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 92
> [junit4] 2> 2152497 DEBUG (ScheduledTrigger-8419-thread-2) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 92
> [junit4] 2> 2152497 DEBUG (ScheduledTrigger-8419-thread-2) [ ]
> o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for
> lost nodes: [127.0.0.1:10052_solr, 127.0.0.1:10105_solr, 127.0.0.1:10026_solr,
> 127.0.0.1:10019_solr]
> [junit4] 2> 2152497 DEBUG (ScheduledTrigger-8419-thread-2) [ ]
> o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
> [junit4] 2> "id":"7c25ef310339Tbb8x7tcy164rfweb08jol5m0l",
> [junit4] 2> "source":"node_lost_trigger",
> [junit4] 2> "eventTime":136502368600889,
> [junit4] 2> "eventType":"NODELOST",
> [junit4] 2> "properties":{
> [junit4] 2> "eventTimes":[
> [junit4] 2> 136502368600889,
> [junit4] 2> 136502363258989,
> [junit4] 2> 136502370026839,
> [junit4] 2> 136502359925389],
> [junit4] 2> "nodeNames":[
> [junit4] 2> "127.0.0.1:10052_solr",
> [junit4] 2> "127.0.0.1:10105_solr",
> [junit4] 2> "127.0.0.1:10026_solr",
> [junit4] 2> "127.0.0.1:10019_solr"]}}
> [junit4] 2> 2152498 DEBUG (ScheduledTrigger-8419-thread-2) [ ]
> o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas,
> node_lost_trigger]
> [junit4] 2> 2152498 DEBUG (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
> [junit4] 2> "id":"7c25ef310339Tbb8x7tcy164rfweb08jol5m0l",
> [junit4] 2> "source":"node_lost_trigger",
> [junit4] 2> "eventTime":136502368600889,
> [junit4] 2> "eventType":"NODELOST",
> [junit4] 2> "properties":{
> [junit4] 2> "eventTimes":[
> [junit4] 2> 136502368600889,
> [junit4] 2> 136502363258989,
> [junit4] 2> 136502370026839,
> [junit4] 2> 136502359925389],
> [junit4] 2> "_enqueue_time_":136507460497839,
> [junit4] 2> "nodeNames":[
> [junit4] 2> "127.0.0.1:10052_solr",
> [junit4] 2> "127.0.0.1:10105_solr",
> [junit4] 2> "127.0.0.1:10026_solr",
> [junit4] 2> "127.0.0.1:10019_solr"]}}
> [junit4] 2> 2152499 DEBUG (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.a.ComputePlanAction -- processing event: {
> [junit4] 2> "id":"7c25ef310339Tbb8x7tcy164rfweb08jol5m0l",
> [junit4] 2> "source":"node_lost_trigger",
> [junit4] 2> "eventTime":136502368600889,
> [junit4] 2> "eventType":"NODELOST",
> [junit4] 2> "properties":{
> [junit4] 2> "eventTimes":[
> [junit4] 2> 136502368600889,
> [junit4] 2> 136502363258989,
> [junit4] 2> 136502370026839,
> [junit4] 2> 136502359925389],
> [junit4] 2> "_enqueue_time_":136507460497839,
> [junit4] 2> "nodeNames":[
> [junit4] 2> "127.0.0.1:10052_solr",
> [junit4] 2> "127.0.0.1:10105_solr",
> [junit4] 2> "127.0.0.1:10026_solr",
> [junit4] 2> "127.0.0.1:10019_solr"]}} with context properties:
> {BEFORE_ACTION=[compute]}
> [junit4] 2> 2152511 INFO (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.a.ComputePlanAction Computed Plan:
> action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.
> 1:10056_solr&inPlaceMove=true&replica=core_node53
> [junit4] 2> 2152518 INFO (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.a.ComputePlanAction Computed Plan:
> action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.
> 1:10072_solr&inPlaceMove=true&replica=core_node40
> [junit4] 2> 2152521 INFO (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.a.ComputePlanAction Computed Plan:
> action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.
> 1:10103_solr&inPlaceMove=true&replica=core_node66
> [junit4] 2> 2152523 INFO (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.a.ComputePlanAction Computed Plan:
> action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.
> 1:10059_solr&inPlaceMove=true&replica=core_node26
> [junit4] 2> 2152523 INFO (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 136508721
> sessionWrapper.createTime 136507569037789, this.sessionWrapper.createTime
> 136507569037789
> [junit4] 2> 2152523 DEBUG (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.a.ExecutePlanAction -- processing event: {
> [junit4] 2> "id":"7c25ef310339Tbb8x7tcy164rfweb08jol5m0l",
> [junit4] 2> "source":"node_lost_trigger",
> [junit4] 2> "eventTime":136502368600889,
> [junit4] 2> "eventType":"NODELOST",
> [junit4] 2> "properties":{
> [junit4] 2> "eventTimes":[
> [junit4] 2> 136502368600889,
> [junit4] 2> 136502363258989,
> [junit4] 2> 136502370026839,
> [junit4] 2> 136502359925389],
> [junit4] 2> "_enqueue_time_":136507460497839,
> [junit4] 2> "nodeNames":[
> [junit4] 2> "127.0.0.1:10052_solr",
> [junit4] 2> "127.0.0.1:10105_solr",
> [junit4] 2> "127.0.0.1:10026_solr",
> [junit4] 2> "127.0.0.1:10019_solr"]}} with context properties:
> {operations=[org.apache.solr.client.solrj.request.CollectionAdminRequest$
> MoveReplica@75c90a70, org.apache.solr.client.solrj.request.
> CollectionAdminRequest$MoveReplica@7b0eeab9, org.apache.solr.client.solrj.
> request.CollectionAdminRequest$MoveReplica@7f316100,
> org.apache.solr.client.solrj.request.CollectionAdminRequest$
> MoveReplica@3a33e082], AFTER_ACTION=[compute], BEFORE_ACTION=[compute,
> execute]}
> [junit4] 2> 2152523 DEBUG (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.a.ExecutePlanAction Executing operation:
> action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.
> 1:10056_solr&inPlaceMove=true&replica=core_node53
> [junit4] 2> 2152624 DEBUG (simCloudManagerPool-8418-thread-120) [
> ] o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node77":{
> [junit4] 2> "core":"testBasic_shard4_replica_t76",
> [junit4] 2> "shard":"shard4",
> [junit4] 2> "collection":"testBasic",
> [junit4] 2> "node_name":"127.0.0.1:10056_solr",
> [junit4] 2> "type":"TLOG"}}
> [junit4] 2> 2152736 DEBUG (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.a.ExecutePlanAction Executing operation:
> action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.
> 1:10072_solr&inPlaceMove=true&replica=core_node40
> [junit4] 2> 2152836 DEBUG (simCloudManagerPool-8418-thread-128) [
> ] o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node79":{
> [junit4] 2> "core":"testBasic_shard3_replica_t78",
> [junit4] 2> "shard":"shard3",
> [junit4] 2> "collection":"testBasic",
> [junit4] 2> "node_name":"127.0.0.1:10072_solr",
> [junit4] 2> "type":"TLOG"}}
> [junit4] 2> 2152948 DEBUG (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.a.ExecutePlanAction Executing operation:
> action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.
> 1:10103_solr&inPlaceMove=true&replica=core_node66
> [junit4] 2> 2153049 DEBUG (simCloudManagerPool-8418-thread-136) [
> ] o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node81":{
> [junit4] 2> "core":"testBasic_shard5_replica_t80",
> [junit4] 2> "shard":"shard5",
> [junit4] 2> "collection":"testBasic",
> [junit4] 2> "node_name":"127.0.0.1:10103_solr",
> [junit4] 2> "type":"TLOG"}}
> [junit4] 2> 2153161 DEBUG (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.a.ExecutePlanAction Executing operation:
> action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.
> 1:10059_solr&inPlaceMove=true&replica=core_node26
> [junit4] 2> 2153261 DEBUG (simCloudManagerPool-8418-thread-144) [
> ] o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node83":{
> [junit4] 2> "core":"testBasic_shard2_replica_p82",
> [junit4] 2> "shard":"shard2",
> [junit4] 2> "collection":"testBasic",
> [junit4] 2> "node_name":"127.0.0.1:10059_solr",
> [junit4] 2> "type":"PULL"}}
> [junit4] 2> 2153373 DEBUG (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas
> after 100ms
> [junit4] 2> 2153373 DEBUG (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_lost_trigger
> after 100ms
> [junit4] 2> 2153373 DEBUG (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.a.ScheduledTriggers -- processing took 43728 ms for event id=
> 7c25ef310339Tbb8x7tcy164rfweb08jol5m0l
> [junit4] 2> 2153473 DEBUG (ScheduledTrigger-8419-thread-1) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 92
> [junit4] 2> 2153473 DEBUG (ScheduledTrigger-8419-thread-3) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 92
> [junit4] 2> 2153473 DEBUG (ScheduledTrigger-8419-thread-1) [ ]
> o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for
> lost nodes: [127.0.0.1:10052_solr, 127.0.0.1:10022_solr, 127.0.0.1:10105_solr,
> 127.0.0.1:10026_solr, 127.0.0.1:10071_solr, 127.0.0.1:10019_solr,
> 127.0.0.1:10036_solr, 127.0.0.1:10076_solr]
> [junit4] 2> 2153473 DEBUG (ScheduledTrigger-8419-thread-3) [ ]
> o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for
> lost nodes: [127.0.0.1:10022_solr, 127.0.0.1:10071_solr, 127.0.0.1:10036_solr,
> 127.0.0.1:10076_solr]
> [junit4] 2> 2153473 DEBUG (ScheduledTrigger-8419-thread-3) [ ]
> o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
> [junit4] 2> "id":"7c262c72b8ffTbb8x7tcy164rfweb08jol5m0p",
> [junit4] 2> "source":"node_lost_trigger",
> [junit4] 2> "eventTime":136503396317439,
> [junit4] 2> "eventType":"NODELOST",
> [junit4] 2> "properties":{
> [junit4] 2> "eventTimes":[
> [junit4] 2> 136503396317439,
> [junit4] 2> 136503401186889,
> [junit4] 2> 136503397793289,
> [junit4] 2> 136503402846389],
> [junit4] 2> "nodeNames":[
> [junit4] 2> "127.0.0.1:10022_solr",
> [junit4] 2> "127.0.0.1:10071_solr",
> [junit4] 2> "127.0.0.1:10036_solr",
> [junit4] 2> "127.0.0.1:10076_solr"]}}
> [junit4] 2> 2153473 DEBUG (ScheduledTrigger-8419-thread-1) [ ]
> o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
> [junit4] 2> "id":"7c25eefb446fTbb8x7tcy164rfweb08jol5m0o",
> [junit4] 2> "source":".auto_add_replicas",
> [junit4] 2> "eventTime":136502365078639,
> [junit4] 2> "eventType":"NODELOST",
> [junit4] 2> "properties":{
> [junit4] 2> "eventTimes":[
> [junit4] 2> 136502365078639,
> [junit4] 2> 136503382704989,
> [junit4] 2> 136502361399939,
> [junit4] 2> 136502366679539,
> [junit4] 2> 136503385718139,
> [junit4] 2> 136502358192739,
> [junit4] 2> 136503384267139,
> [junit4] 2> 136503387178989],
> [junit4] 2> "nodeNames":[
> [junit4] 2> "127.0.0.1:10052_solr",
> [junit4] 2> "127.0.0.1:10022_solr",
> [junit4] 2> "127.0.0.1:10105_solr",
> [junit4] 2> "127.0.0.1:10026_solr",
> [junit4] 2> "127.0.0.1:10071_solr",
> [junit4] 2> "127.0.0.1:10019_solr",
> [junit4] 2> "127.0.0.1:10036_solr",
> [junit4] 2> "127.0.0.1:10076_solr"]}}
> [junit4] 2> 2153473 DEBUG (ScheduledTrigger-8419-thread-3) [ ]
> o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas,
> node_lost_trigger]
> [junit4] 2> 2153474 DEBUG (ScheduledTrigger-8419-thread-1) [ ]
> o.a.s.c.a.NodeLostTrigger NodeLostTrigger processor for lost nodes:
> [127.0.0.1:10052_solr, 127.0.0.1:10022_solr, 127.0.0.1:10105_solr,
> 127.0.0.1:10026_solr, 127.0.0.1:10071_solr, 127.0.0.1:10019_solr,
> 127.0.0.1:10036_solr, 127.0.0.1:10076_solr] is not ready, will try later
> [junit4] 2> 2153474 DEBUG (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
> [junit4] 2> "id":"7c262c72b8ffTbb8x7tcy164rfweb08jol5m0p",
> [junit4] 2> "source":"node_lost_trigger",
> [junit4] 2> "eventTime":136503396317439,
> [junit4] 2> "eventType":"NODELOST",
> [junit4] 2> "properties":{
> [junit4] 2> "eventTimes":[
> [junit4] 2> 136503396317439,
> [junit4] 2> 136503401186889,
> [junit4] 2> 136503397793289,
> [junit4] 2> 136503402846389],
> [junit4] 2> "_enqueue_time_":136556253282539,
> [junit4] 2> "nodeNames":[
> [junit4] 2> "127.0.0.1:10022_solr",
> [junit4] 2> "127.0.0.1:10071_solr",
> [junit4] 2> "127.0.0.1:10036_solr",
> [junit4] 2> "127.0.0.1:10076_solr"]}}
> [junit4] 2> 2153474 DEBUG (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.a.ComputePlanAction -- processing event: {
> [junit4] 2> "id":"7c262c72b8ffTbb8x7tcy164rfweb08jol5m0p",
> [junit4] 2> "source":"node_lost_trigger",
> [junit4] 2> "eventTime":136503396317439,
> [junit4] 2> "eventType":"NODELOST",
> [junit4] 2> "properties":{
> [junit4] 2> "eventTimes":[
> [junit4] 2> 136503396317439,
> [junit4] 2> 136503401186889,
> [junit4] 2> 136503397793289,
> [junit4] 2> 136503402846389],
> [junit4] 2> "_enqueue_time_":136556253282539,
> [junit4] 2> "nodeNames":[
> [junit4] 2> "127.0.0.1:10022_solr",
> [junit4] 2> "127.0.0.1:10071_solr",
> [junit4] 2> "127.0.0.1:10036_solr",
> [junit4] 2> "127.0.0.1:10076_solr"]}} with context properties:
> {BEFORE_ACTION=[compute]}
> [junit4] 2> 2153482 INFO (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.a.ComputePlanAction Computed Plan:
> action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.
> 1:10053_solr&inPlaceMove=true&replica=core_node70
> [junit4] 2> 2153487 INFO (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.a.ComputePlanAction Computed Plan:
> action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.
> 1:10109_solr&inPlaceMove=true&replica=core_node17
> [junit4] 2> 2153490 INFO (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.a.ComputePlanAction Computed Plan:
> action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.
> 1:10037_solr&inPlaceMove=true&replica=core_node48
> [junit4] 2> 2153492 INFO (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.a.ComputePlanAction Computed Plan:
> action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.
> 1:10100_solr&inPlaceMove=true&replica=core_node23
> [junit4] 2> 2153492 INFO (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 136557194
> sessionWrapper.createTime 136556333599339, this.sessionWrapper.createTime
> 136556333599339
> [junit4] 2> 2153493 DEBUG (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.a.ExecutePlanAction -- processing event: {
> [junit4] 2> "id":"7c262c72b8ffTbb8x7tcy164rfweb08jol5m0p",
> [junit4] 2> "source":"node_lost_trigger",
> [junit4] 2> "eventTime":136503396317439,
> [junit4] 2> "eventType":"NODELOST",
> [junit4] 2> "properties":{
> [junit4] 2> "eventTimes":[
> [junit4] 2> 136503396317439,
> [junit4] 2> 136503401186889,
> [junit4] 2> 136503397793289,
> [junit4] 2> 136503402846389],
> [junit4] 2> "_enqueue_time_":136556253282539,
> [junit4] 2> "nodeNames":[
> [junit4] 2> "127.0.0.1:10022_solr",
> [junit4] 2> "127.0.0.1:10071_solr",
> [junit4] 2> "127.0.0.1:10036_solr",
> [junit4] 2> "127.0.0.1:10076_solr"]}} with context properties:
> {operations=[org.apache.solr.client.solrj.request.CollectionAdminRequest$
> MoveReplica@37a7a747, org.apache.solr.client.solrj.request.
> CollectionAdminRequest$MoveReplica@4b5c2da8, org.apache.solr.client.solrj.
> request.CollectionAdminRequest$MoveReplica@3d0e4875,
> org.apache.solr.client.solrj.request.CollectionAdminRequest$
> MoveReplica@5045cd61], AFTER_ACTION=[compute], BEFORE_ACTION=[compute,
> execute]}
> [junit4] 2> 2153493 DEBUG (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.a.ExecutePlanAction Executing operation:
> action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.
> 1:10053_solr&inPlaceMove=true&replica=core_node70
> [junit4] 2> 2153593 DEBUG (simCloudManagerPool-8418-thread-155) [
> ] o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node85":{
> [junit4] 2> "core":"testBasic_shard5_replica_t84",
> [junit4] 2> "shard":"shard5",
> [junit4] 2> "collection":"testBasic",
> [junit4] 2> "node_name":"127.0.0.1:10053_solr",
> [junit4] 2> "type":"TLOG"}}
> [junit4] 2> 2153705 DEBUG (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.a.ExecutePlanAction Executing operation:
> action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.
> 1:10109_solr&inPlaceMove=true&replica=core_node17
> [junit4] 2> 2153805 DEBUG (simCloudManagerPool-8418-thread-163) [
> ] o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node87":{
> [junit4] 2> "core":"testBasic_shard2_replica_n86",
> [junit4] 2> "shard":"shard2",
> [junit4] 2> "collection":"testBasic",
> [junit4] 2> "node_name":"127.0.0.1:10109_solr",
> [junit4] 2> "type":"NRT"}}
> [junit4] 2> 2153917 DEBUG (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.a.ExecutePlanAction Executing operation:
> action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.
> 1:10037_solr&inPlaceMove=true&replica=core_node48
> [junit4] 2> 2154017 DEBUG (simCloudManagerPool-8418-thread-171) [
> ] o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node89":{
> [junit4] 2> "core":"testBasic_shard4_replica_n88",
> [junit4] 2> "shard":"shard4",
> [junit4] 2> "collection":"testBasic",
> [junit4] 2> "node_name":"127.0.0.1:10037_solr",
> [junit4] 2> "type":"NRT"}}
> [junit4] 2> 2154129 DEBUG (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.a.ExecutePlanAction Executing operation:
> action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.
> 1:10100_solr&inPlaceMove=true&replica=core_node23
> [junit4] 2> 2154230 DEBUG (simCloudManagerPool-8418-thread-179) [
> ] o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node91":{
> [junit4] 2> "core":"testBasic_shard2_replica_t90",
> [junit4] 2> "shard":"shard2",
> [junit4] 2> "collection":"testBasic",
> [junit4] 2> "node_name":"127.0.0.1:10100_solr",
> [junit4] 2> "type":"TLOG"}}
> [junit4] 2> 2154341 INFO (TEST-TestLargeCluster.testBasic-seed#[8A7B7719C587386D])
> [ ] o.a.s.c.a.s.TestLargeCluster Ready after 96333ms
> [junit4] 2> 2154341 DEBUG (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas
> after 100ms
> [junit4] 2> 2154342 DEBUG (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_lost_trigger
> after 100ms
> [junit4] 2> 2154342 INFO (TEST-TestLargeCluster.testBasic-seed#[8A7B7719C587386D])
> [ ] o.a.s.c.a.s.TestLargeCluster OP COUNTS: {CREATE=1, DELETESTATUS=8,
> MOVEREPLICA=8, REQUESTSTATUS=8, autoscaling=2, update=5}
> [junit4] 2> 2154342 DEBUG (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.a.ScheduledTriggers -- processing took 43379 ms for event id=
> 7c262c72b8ffTbb8x7tcy164rfweb08jol5m0p
> [junit4] 2> 2154345 DEBUG (simCloudManagerPool-8418-thread-8) [ ]
> o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testBasic /
> shard5: {"core_node64":{
> [junit4] 2> "node_name":"127.0.0.1:10047_solr",
> [junit4] 2> "core":"testBasic_shard5_replica_n64",
> [junit4] 2> "leader":"true",
> [junit4] 2> "state":"active",
> [junit4] 2> "type":"NRT",
> [junit4] 2> "INDEX.sizeInBytes":123450000,
> [junit4] 2> "shard":"shard5",
> [junit4] 2> "collection":"testBasic"}}
> [junit4] 2> 2154442 DEBUG (ScheduledTrigger-8419-thread-4) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 82
> [junit4] 2> 2154442 DEBUG (ScheduledTrigger-8419-thread-2) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 82
> [junit4] 2> 2154442 DEBUG (ScheduledTrigger-8419-thread-4) [ ]
> o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10018_solr
> [junit4] 2> 2154442 DEBUG (ScheduledTrigger-8419-thread-2) [ ]
> o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10018_solr
> [junit4] 2> 2154442 DEBUG (ScheduledTrigger-8419-thread-4) [ ]
> o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10049_solr
> [junit4] 2> 2154442 DEBUG (ScheduledTrigger-8419-thread-2) [ ]
> o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10049_solr
> [junit4] 2> 2154442 DEBUG (ScheduledTrigger-8419-thread-4) [ ]
> o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10074_solr
> [junit4] 2> 2154442 DEBUG (ScheduledTrigger-8419-thread-2) [ ]
> o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10074_solr
> [junit4] 2> 2154442 DEBUG (ScheduledTrigger-8419-thread-4) [ ]
> o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10113_solr
> [junit4] 2> 2154442 DEBUG (ScheduledTrigger-8419-thread-2) [ ]
> o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10113_solr
> [junit4] 2> 2154442 DEBUG (ScheduledTrigger-8419-thread-4) [ ]
> o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10035_solr
> [junit4] 2> 2154442 DEBUG (ScheduledTrigger-8419-thread-2) [ ]
> o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10035_solr
> [junit4] 2> 2154442 DEBUG (ScheduledTrigger-8419-thread-4) [ ]
> o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10087_solr
> [junit4] 2> 2154442 DEBUG (ScheduledTrigger-8419-thread-2) [ ]
> o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10087_solr
> [junit4] 2> 2154442 DEBUG (ScheduledTrigger-8419-thread-2) [ ]
> o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10085_solr
> [junit4] 2> 2154442 DEBUG (ScheduledTrigger-8419-thread-4) [ ]
> o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10085_solr
> [junit4] 2> 2154442 DEBUG (ScheduledTrigger-8419-thread-2) [ ]
> o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10046_solr
> [junit4] 2> 2154442 DEBUG (ScheduledTrigger-8419-thread-4) [ ]
> o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10046_solr
> [junit4] 2> 2154442 DEBUG (ScheduledTrigger-8419-thread-2) [ ]
> o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10038_solr
> [junit4] 2> 2154442 DEBUG (ScheduledTrigger-8419-thread-4) [ ]
> o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10038_solr
> [junit4] 2> 2154442 DEBUG (ScheduledTrigger-8419-thread-2) [ ]
> o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10086_solr
> [junit4] 2> 2154442 DEBUG (ScheduledTrigger-8419-thread-4) [ ]
> o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10086_solr
> [junit4] 2> 2154442 DEBUG (ScheduledTrigger-8419-thread-2) [ ]
> o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for
> lost nodes: [127.0.0.1:10022_solr, 127.0.0.1:10105_solr, 127.0.0.1:10026_solr,
> 127.0.0.1:10071_solr, 127.0.0.1:10036_solr, 127.0.0.1:10076_solr,
> 127.0.0.1:10052_solr, 127.0.0.1:10019_solr]
> [junit4] 2> 2154443 DEBUG (ScheduledTrigger-8419-thread-2) [ ]
> o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
> [junit4] 2> "id":"7c262ba3035dTbb8x7tcy164rfweb08jol5m0t",
> [junit4] 2> "source":".auto_add_replicas",
> [junit4] 2> "eventTime":136503382704989,
> [junit4] 2> "eventType":"NODELOST",
> [junit4] 2> "properties":{
> [junit4] 2> "eventTimes":[
> [junit4] 2> 136503382704989,
> [junit4] 2> 136502361399939,
> [junit4] 2> 136502366679539,
> [junit4] 2> 136503385718139,
> [junit4] 2> 136503384267139,
> [junit4] 2> 136503387178989,
> [junit4] 2> 136502365078639,
> [junit4] 2> 136502358192739],
> [junit4] 2> "nodeNames":[
> [junit4] 2> "127.0.0.1:10022_solr",
> [junit4] 2> "127.0.0.1:10105_solr",
> [junit4] 2> "127.0.0.1:10026_solr",
> [junit4] 2> "127.0.0.1:10071_solr",
> [junit4] 2> "127.0.0.1:10036_solr",
> [junit4] 2> "127.0.0.1:10076_solr",
> [junit4] 2> "127.0.0.1:10052_solr",
> [junit4] 2> "127.0.0.1:10019_solr"]}}
> [junit4] 2> 2154443 DEBUG (ScheduledTrigger-8419-thread-2) [ ]
> o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas,
> node_lost_trigger]
> [junit4] 2> 2154443 DEBUG (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
> [junit4] 2> "id":"7c262ba3035dTbb8x7tcy164rfweb08jol5m0t",
> [junit4] 2> "source":".auto_add_replicas",
> [junit4] 2> "eventTime":136503382704989,
> [junit4] 2> "eventType":"NODELOST",
> [junit4] 2> "properties":{
> [junit4] 2> "eventTimes":[
> [junit4] 2> 136503382704989,
> [junit4] 2> 136502361399939,
> [junit4] 2> 136502366679539,
> [junit4] 2> 136503385718139,
> [junit4] 2> 136503384267139,
> [junit4] 2> 136503387178989,
> [junit4] 2> 136502365078639,
> [junit4] 2> 136502358192739],
> [junit4] 2> "_enqueue_time_":136604710274189,
> [junit4] 2> "nodeNames":[
> [junit4] 2> "127.0.0.1:10022_solr",
> [junit4] 2> "127.0.0.1:10105_solr",
> [junit4] 2> "127.0.0.1:10026_solr",
> [junit4] 2> "127.0.0.1:10071_solr",
> [junit4] 2> "127.0.0.1:10036_solr",
> [junit4] 2> "127.0.0.1:10076_solr",
> [junit4] 2> "127.0.0.1:10052_solr",
> [junit4] 2> "127.0.0.1:10019_solr"]}}
> [junit4] 2> 2154443 DEBUG (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.a.ComputePlanAction -- processing event: {
> [junit4] 2> "id":"7c262ba3035dTbb8x7tcy164rfweb08jol5m0t",
> [junit4] 2> "source":".auto_add_replicas",
> [junit4] 2> "eventTime":136503382704989,
> [junit4] 2> "eventType":"NODELOST",
> [junit4] 2> "properties":{
> [junit4] 2> "eventTimes":[
> [junit4] 2> 136503382704989,
> [junit4] 2> 136502361399939,
> [junit4] 2> 136502366679539,
> [junit4] 2> 136503385718139,
> [junit4] 2> 136503384267139,
> [junit4] 2> 136503387178989,
> [junit4] 2> 136502365078639,
> [junit4] 2> 136502358192739],
> [junit4] 2> "_enqueue_time_":136604710274189,
> [junit4] 2> "nodeNames":[
> [junit4] 2> "127.0.0.1:10022_solr",
> [junit4] 2> "127.0.0.1:10105_solr",
> [junit4] 2> "127.0.0.1:10026_solr",
> [junit4] 2> "127.0.0.1:10071_solr",
> [junit4] 2> "127.0.0.1:10036_solr",
> [junit4] 2> "127.0.0.1:10076_solr",
> [junit4] 2> "127.0.0.1:10052_solr",
> [junit4] 2> "127.0.0.1:10019_solr"]}} with context properties:
> {BEFORE_ACTION=[auto_add_replicas_plan]}
> [junit4] 2> 2154445 INFO (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 136604805
> sessionWrapper.createTime 136604799251589, this.sessionWrapper.createTime
> 136604799251589
> [junit4] 2> 2154445 DEBUG (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.a.ExecutePlanAction -- processing event: {
> [junit4] 2> "id":"7c262ba3035dTbb8x7tcy164rfweb08jol5m0t",
> [junit4] 2> "source":".auto_add_replicas",
> [junit4] 2> "eventTime":136503382704989,
> [junit4] 2> "eventType":"NODELOST",
> [junit4] 2> "properties":{
> [junit4] 2> "eventTimes":[
> [junit4] 2> 136503382704989,
> [junit4] 2> 136502361399939,
> [junit4] 2> 136502366679539,
> [junit4] 2> 136503385718139,
> [junit4] 2> 136503384267139,
> [junit4] 2> 136503387178989,
> [junit4] 2> 136502365078639,
> [junit4] 2> 136502358192739],
> [junit4] 2> "_enqueue_time_":136604710274189,
> [junit4] 2> "nodeNames":[
> [junit4] 2> "127.0.0.1:10022_solr",
> [junit4] 2> "127.0.0.1:10105_solr",
> [junit4] 2> "127.0.0.1:10026_solr",
> [junit4] 2> "127.0.0.1:10071_solr",
> [junit4] 2> "127.0.0.1:10036_solr",
> [junit4] 2> "127.0.0.1:10076_solr",
> [junit4] 2> "127.0.0.1:10052_solr",
> [junit4] 2> "127.0.0.1:10019_solr"]}} with context properties:
> {AFTER_ACTION=[auto_add_replicas_plan], BEFORE_ACTION=[auto_add_replicas_plan,
> execute_plan]}
> [junit4] 2> 2154445 INFO (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.a.ExecutePlanAction No operations to execute for event: {
> [junit4] 2> "id":"7c262ba3035dTbb8x7tcy164rfweb08jol5m0t",
> [junit4] 2> "source":".auto_add_replicas",
> [junit4] 2> "eventTime":136503382704989,
> [junit4] 2> "eventType":"NODELOST",
> [junit4] 2> "properties":{
> [junit4] 2> "eventTimes":[
> [junit4] 2> 136503382704989,
> [junit4] 2> 136502361399939,
> [junit4] 2> 136502366679539,
> [junit4] 2> 136503385718139,
> [junit4] 2> 136503384267139,
> [junit4] 2> 136503387178989,
> [junit4] 2> 136502365078639,
> [junit4] 2> 136502358192739],
> [junit4] 2> "_enqueue_time_":136604710274189,
> [junit4] 2> "nodeNames":[
> [junit4] 2> "127.0.0.1:10022_solr",
> [junit4] 2> "127.0.0.1:10105_solr",
> [junit4] 2> "127.0.0.1:10026_solr",
> [junit4] 2> "127.0.0.1:10071_solr",
> [junit4] 2> "127.0.0.1:10036_solr",
> [junit4] 2> "127.0.0.1:10076_solr",
> [junit4] 2> "127.0.0.1:10052_solr",
> [junit4] 2> "127.0.0.1:10019_solr"]}}
> [junit4] 2> 2154445 DEBUG (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas
> after 100ms
> [junit4] 2> 2154445 DEBUG (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_lost_trigger
> after 100ms
> [junit4] 2> 2154445 DEBUG (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.a.ScheduledTriggers -- processing took 115 ms for event id=
> 7c262ba3035dTbb8x7tcy164rfweb08jol5m0t
> [junit4] 2> 2154545 DEBUG (ScheduledTrigger-8419-thread-1) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with
> currently live nodes: 82
> [junit4] 2> 2154545 DEBUG (ScheduledTrigger-8419-thread-3) [ ]
> o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
> currently live nodes: 82
> [junit4] 2> 2154545 DEBUG (ScheduledTrigger-8419-thread-1) [ ]
> o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for
> lost nodes: [127.0.0.1:10085_solr, 127.0.0.1:10018_solr, 127.0.0.1:10049_solr,
> 127.0.0.1:10087_solr, 127.0.0.1:10074_solr, 127.0.0.1:10046_solr,
> 127.0.0.1:10113_solr, 127.0.0.1:10035_solr, 127.0.0.1:10038_solr,
> 127.0.0.1:10086_solr]
> [junit4] 2> 2154546 DEBUG (ScheduledTrigger-8419-thread-1) [ ]
> o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
> [junit4] 2> "id":"7c3dc1fda675Tbb8x7tcy164rfweb08jol5m0w",
> [junit4] 2> "source":"node_lost_trigger",
> [junit4] 2> "eventTime":136604689475189,
> [junit4] 2> "eventType":"NODELOST",
> [junit4] 2> "properties":{
> [junit4] 2> "eventTimes":[
> [junit4] 2> 136604689475189,
> [junit4] 2> 136604668565089,
> [junit4] 2> 136604671986439,
> [junit4] 2> 136604684703439,
> [junit4] 2> 136604675170889,
> [junit4] 2> 136604692307289,
> [junit4] 2> 136604678305839,
> [junit4] 2> 136604681461739,
> [junit4] 2> 136604695299239,
> [junit4] 2> 136604698257039],
> [junit4] 2> "nodeNames":[
> [junit4] 2> "127.0.0.1:10085_solr",
> [junit4] 2> "127.0.0.1:10018_solr",
> [junit4] 2> "127.0.0.1:10049_solr",
> [junit4] 2> "127.0.0.1:10087_solr",
> [junit4] 2> "127.0.0.1:10074_solr",
> [junit4] 2> "127.0.0.1:10046_solr",
> [junit4] 2> "127.0.0.1:10113_solr",
> [junit4] 2> "127.0.0.1:10035_solr",
> [junit4] 2> "127.0.0.1:10038_solr",
> [junit4] 2> "127.0.0.1:10086_solr"]}}
> [junit4] 2> 2154546 DEBUG (ScheduledTrigger-8419-thread-1) [ ]
> o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas,
> node_lost_trigger]
> [junit4] 2> 2154546 DEBUG (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
> [junit4] 2> "id":"7c3dc1fda675Tbb8x7tcy164rfweb08jol5m0w",
> [junit4] 2> "source":"node_lost_trigger",
> [junit4] 2> "eventTime":136604689475189,
> [junit4] 2> "eventType":"NODELOST",
> [junit4] 2> "properties":{
> [junit4] 2> "eventTimes":[
> [junit4] 2> 136604689475189,
> [junit4] 2> 136604668565089,
> [junit4] 2> 136604671986439,
> [junit4] 2> 136604684703439,
> [junit4] 2> 136604675170889,
> [junit4] 2> 136604692307289,
> [junit4] 2> 136604678305839,
> [junit4] 2> 136604681461739,
> [junit4] 2> 136604695299239,
> [junit4] 2> 136604698257039],
> [junit4] 2> "_enqueue_time_":136609861116439,
> [junit4] 2> "nodeNames":[
> [junit4] 2> "127.0.0.1:10085_solr",
> [junit4] 2> "127.0.0.1:10018_solr",
> [junit4] 2> "127.0.0.1:10049_solr",
> [junit4] 2> "127.0.0.1:10087_solr",
> [junit4] 2> "127.0.0.1:10074_solr",
> [junit4] 2> "127.0.0.1:10046_solr",
> [junit4] 2> "127.0.0.1:10113_solr",
> [junit4] 2> "127.0.0.1:10035_solr",
> [junit4] 2> "127.0.0.1:10038_solr",
> [junit4] 2> "127.0.0.1:10086_solr"]}}
> [junit4] 2> 2154546 DEBUG (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.a.ComputePlanAction -- processing event: {
> [junit4] 2> "id":"7c3dc1fda675Tbb8x7tcy164rfweb08jol5m0w",
> [junit4] 2> "source":"node_lost_trigger",
> [junit4] 2> "eventTime":136604689475189,
> [junit4] 2> "eventType":"NODELOST",
> [junit4] 2> "properties":{
> [junit4] 2> "eventTimes":[
> [junit4] 2> 136604689475189,
> [junit4] 2> 136604668565089,
> [junit4] 2> 136604671986439,
> [junit4] 2> 136604684703439,
> [junit4] 2> 136604675170889,
> [junit4] 2> 136604692307289,
> [junit4] 2> 136604678305839,
> [junit4] 2> 136604681461739,
> [junit4] 2> 136604695299239,
> [junit4] 2> 136604698257039],
> [junit4] 2> "_enqueue_time_":136609861116439,
> [junit4] 2> "nodeNames":[
> [junit4] 2> "127.0.0.1:10085_solr",
> [junit4] 2> "127.0.0.1:10018_solr",
> [junit4] 2> "127.0.0.1:10049_solr",
> [junit4] 2> "127.0.0.1:10087_solr",
> [junit4] 2> "127.0.0.1:10074_solr",
> [junit4] 2> "127.0.0.1:10046_solr",
> [junit4] 2> "127.0.0.1:10113_solr",
> [junit4] 2> "127.0.0.1:10035_solr",
> [junit4] 2> "127.0.0.1:10038_solr",
> [junit4] 2> "127.0.0.1:10086_solr"]}} with context properties:
> {BEFORE_ACTION=[compute]}
> [junit4] 2> 2154560 INFO (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.a.ComputePlanAction Computed Plan:
> action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.
> 1:10106_solr&inPlaceMove=true&replica=core_node41
> [junit4] 2> 2154571 INFO (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.a.ComputePlanAction Computed Plan:
> action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.
> 1:10094_solr&inPlaceMove=true&replica=core_node5
> [junit4] 2> 2154582 INFO (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.a.ComputePlanAction Computed Plan:
> action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.
> 1:10091_solr&inPlaceMove=true&replica=core_node72
> [junit4] 2> 2154591 INFO (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.a.ComputePlanAction Computed Plan:
> action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.
> 1:10034_solr&inPlaceMove=true&replica=core_node73
> [junit4] 2> 2154598 INFO (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.a.ComputePlanAction Computed Plan:
> action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.
> 1:10031_solr&inPlaceMove=true&replica=core_node10
> [junit4] 2> 2154605 INFO (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.a.ComputePlanAction Computed Plan:
> action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.
> 1:10097_solr&inPlaceMove=true&replica=core_node74
> [junit4] 2> 2154610 INFO (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.a.ComputePlanAction Computed Plan:
> action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.
> 1:10078_solr&inPlaceMove=true&replica=core_node24
> [junit4] 2> 2154614 INFO (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.a.ComputePlanAction Computed Plan:
> action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.
> 1:10075_solr&inPlaceMove=true&replica=core_node65
> [junit4] 2> 2154617 INFO (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.a.ComputePlanAction Computed Plan:
> action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.
> 1:10050_solr&inPlaceMove=true&replica=core_node43
> [junit4] 2> 2154619 INFO (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.a.ComputePlanAction Computed Plan:
> action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.
> 1:10024_solr&inPlaceMove=true&replica=core_node42
> [junit4] 2> 2154619 INFO (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 136613539
> sessionWrapper.createTime 136609921070689, this.sessionWrapper.createTime
> 136609921070689
> [junit4] 2> 2154619 DEBUG (AutoscalingActionExecutor-8420-thread-1)
> [ ] o.a.s.c.a.ExecutePlanAction -- processing event: {
> [junit4] 2> "id":"7c3dc1fda675Tbb8x7tcy164rfweb08jol5m0w",
> [junit4] 2> "source":"node_lost_trigger",
> [junit4] 2> "eventTime":136604689475189,
> [junit4] 2> "eventType":"NODELOST",
> [junit4] 2> "properties":{
> [junit4] 2> "eventTimes":[
> [junit4] 2> 136604689475189,
> [junit4] 2> 136604668565089,
> [junit4] 2> 136604671986439,
> [junit4] 2> 136604684703439,
> [junit4] 2> 136604675170889,
> [junit4] 2> 136604692307289,
> [junit4] 2> 136604678305839,
> [junit4] 2> 136604681461739,
> [junit4] 2> 136604695299239,
> [junit4] 2> 136604698257039],
> [junit4] 2> "_enqueue_time_":136609861116439,
> [junit4] 2> "nodeNames":[
> [junit4] 2> "127.0.0.1:10085_solr",
> [junit4] 2> "127.0.0.1:10018_solr",
> [junit4] 2> "127.0.0.1:10049_solr",
> [junit4] 2> "127.0.0.1:10087_solr",
> [junit4] 2> "127.0.0.1:10074_solr",
> [junit4] 2> "127.0.0.1:10046_solr",
> [junit4] 2> "127.0.0.1:10113_solr",
> [junit4] 2> "127.0.0.1:10035_solr",
> [junit4] 2> "127.0.0.1:10038_solr",
> [junit4] 2> "127.0.0.1:10086_solr"]}} with context properties:
> {operations=[org.apache.solr.client.solrj.request.CollectionAdminRequest$
> MoveReplica@1b6a6404, org.apache.solr.client.solrj.request.
> CollectionAdminRequest$MoveReplica@3ab062da, org.apache.solr.client.solrj.
> request.CollectionAdminRequest$MoveReplica@102f997e,
> org.apache.solr.client.solrj.request.CollectionAdminRequest$
> MoveReplica@2213cac8, org.apache.solr.client.solrj.request.
> CollectionAdminRequest$MoveReplica@6a11c0ca, org.apache.solr.client.solrj.
> request.CollectionAdminRequest$MoveReplica@538ed9e0,
> org.apache.solr.client.solrj.request.CollectionAdminRequest$
> MoveReplica@3f865e86, org.apache.solr.client.so
>
> [...truncated too long message...]
>
> rter [org.apache.solr.metrics.reporters.SolrJmxReporter@2549944e:
> rootName = solr_32851, domain = solr.jetty, service url = null, agent id =
> null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@b3e55e
> [junit4] 2> 203839 INFO (jetty-closer-202-thread-4) [ ]
> o.a.s.c.Overseer Overseer (id=99400986694778892-127.0.0.1:44601_solr-n_0000000000)
> closing
> [junit4] 2> 203839 INFO (OverseerStateUpdate-
> 99400986694778892-127.0.0.1:44601_solr-n_0000000000) [n:127.0.0.1:44601_solr
> ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:44601_solr
> [junit4] 2> 203839 WARN (OverseerAutoScalingTriggerThre
> ad-99400986694778892-127.0.0.1:44601_solr-n_0000000000) [n:127.0.0.1:44601_solr
> ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we
> are closed, exiting.
> [junit4] 2> 203839 INFO (jetty-closer-202-thread-2) [ ]
> o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.
> 1:32851_solr
> [junit4] 2> 203840 INFO (jetty-closer-202-thread-2) [ ]
> o.a.s.m.SolrMetricManager Closing metric reporters for
> registry=solr.cluster, tag=null
> [junit4] 2> 203840 INFO (zkCallback-231-thread-1-
> processing-n:127.0.0.1:44601_solr) [n:127.0.0.1:44601_solr ]
> o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
> [junit4] 2> 203840 INFO (zkCallback-233-thread-1-
> processing-n:127.0.0.1:34381_solr) [n:127.0.0.1:34381_solr ]
> o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
> [junit4] 2> 203840 INFO (zkCallback-229-thread-1-
> processing-n:127.0.0.1:32851_solr) [n:127.0.0.1:32851_solr ]
> o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
> [junit4] 2> 203840 INFO (zkCallback-1958-thread-1-
> processing-n:127.0.0.1:34381_solr x:collection1_shard1_replica_n1
> s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:34381_solr
> c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1]
> o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
> [junit4] 2> 203841 INFO (coreCloseExecutor-729-thread-1)
> [n:127.0.0.1:34381_solr c:collection1 s:shard1 r:core_node3
> x:collection1_shard1_replica_n1] o.a.s.c.SolrCore
> [collection1_shard1_replica_n1] CLOSING SolrCore
> org.apache.solr.core.SolrCore@3e5ef7e8
> [junit4] 2> 203842 INFO (zkCallback-233-thread-1-
> processing-n:127.0.0.1:34381_solr) [n:127.0.0.1:34381_solr ] o.a.s.c.OverseerElectionContext
> I am going to be the leader 127.0.0.1:34381_solr
> [junit4] 2> 203842 INFO (jetty-closer-202-thread-4) [ ]
> o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@
> f4f9736{/solr,null,UNAVAILABLE}
> [junit4] 2> 203843 INFO (jetty-closer-202-thread-4) [ ]
> o.e.j.s.session Stopped scavenging
> [junit4] 2> 203845 INFO (coreCloseExecutor-729-thread-1)
> [n:127.0.0.1:34381_solr c:collection1 s:shard1 r:core_node3
> x:collection1_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric
> reporters for registry=solr.core.collection1.shard1.replica_n1,
> tag=1046411240
> [junit4] 2> 203845 INFO (coreCloseExecutor-729-thread-1)
> [n:127.0.0.1:34381_solr c:collection1 s:shard1 r:core_node3
> x:collection1_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing
> reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@3c1fb8e8:
> rootName = solr_34381, domain = solr.core.collection1.shard1.replica_n1,
> service url = null, agent id = null] for registry
> solr.core.collection1.shard1.replica_n1 / com.codahale.metrics.
> MetricRegistry@5fc54476
> [junit4] 2> 203849 INFO (coreCloseExecutor-731-thread-1)
> [n:127.0.0.1:32851_solr c:collection1 s:shard2 r:core_node4
> x:collection1_shard2_replica_n2] o.a.s.c.SolrCore
> [collection1_shard2_replica_n2] CLOSING SolrCore
> org.apache.solr.core.SolrCore@30fe20c2
> [junit4] 2> 203849 INFO (coreCloseExecutor-731-thread-1)
> [n:127.0.0.1:32851_solr c:collection1 s:shard2 r:core_node4
> x:collection1_shard2_replica_n2] o.a.s.m.SolrMetricManager Closing metric
> reporters for registry=solr.core.collection1.shard2.replica_n2,
> tag=821960898
> [junit4] 2> 203849 INFO (coreCloseExecutor-731-thread-1)
> [n:127.0.0.1:32851_solr c:collection1 s:shard2 r:core_node4
> x:collection1_shard2_replica_n2] o.a.s.m.r.SolrJmxReporter Closing
> reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@4a8598e6:
> rootName = solr_32851, domain = solr.core.collection1.shard2.replica_n2,
> service url = null, agent id = null] for registry
> solr.core.collection1.shard2.replica_n2 / com.codahale.metrics.
> MetricRegistry@39f70898
> [junit4] 2> 203850 INFO (coreCloseExecutor-729-thread-1)
> [n:127.0.0.1:34381_solr c:collection1 s:shard1 r:core_node3
> x:collection1_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric
> reporters for registry=solr.collection.collection1.shard1.leader,
> tag=1046411240
> [junit4] 2> 203852 INFO (zkCallback-229-thread-1-
> processing-n:127.0.0.1:32851_solr) [n:127.0.0.1:32851_solr ] o.a.s.c.OverseerElectionContext
> I am going to be the leader 127.0.0.1:32851_solr
> [junit4] 2> 203852 INFO (jetty-closer-202-thread-1) [ ]
> o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@
> 23f35715{/solr,null,UNAVAILABLE}
> [junit4] 2> 203853 INFO (jetty-closer-202-thread-1) [ ]
> o.e.j.s.session Stopped scavenging
> [junit4] 2> 203856 INFO (coreCloseExecutor-731-thread-1)
> [n:127.0.0.1:32851_solr c:collection1 s:shard2 r:core_node4
> x:collection1_shard2_replica_n2] o.a.s.m.SolrMetricManager Closing metric
> reporters for registry=solr.collection.collection1.shard2.leader,
> tag=821960898
> [junit4] 2> 203857 INFO (jetty-closer-202-thread-2) [ ]
> o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@
> 1ec279a9{/solr,null,UNAVAILABLE}
> [junit4] 2> 203857 INFO (jetty-closer-202-thread-2) [ ]
> o.e.j.s.session Stopped scavenging
> [junit4] 2> 205335 WARN (zkCallback-235-thread-2-
> processing-n:127.0.0.1:37123_solr) [n:127.0.0.1:37123_solr ]
> o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to
> ZK: [KeeperErrorCode = Session expired for /live_nodes]
> [junit4] 2> 205335 INFO (jetty-closer-202-thread-3) [ ]
> o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@
> 1e0d12f{/solr,null,UNAVAILABLE}
> [junit4] 2> 205335 INFO (jetty-closer-202-thread-3) [ ]
> o.e.j.s.session Stopped scavenging
> [junit4] 2> 205336 ERROR (SUITE-StreamExpressionTest-
> seed#[6471768B25648F68]-worker) [ ] o.a.z.s.ZooKeeperServer
> ZKShutdownHandler is not registered, so ZooKeeper server won't take any
> action on ERROR or SHUTDOWN server state changes
> [junit4] 2> 205336 INFO (SUITE-StreamExpressionTest-
> seed#[6471768B25648F68]-worker) [ ] o.a.s.c.ZkTestServer connecting to
> 127.0.0.1:42981 42981
> [junit4] 2> 210104 INFO (Thread-144) [ ] o.a.s.c.ZkTestServer
> connecting to 127.0.0.1:42981 42981
> [junit4] 2> 210104 WARN (Thread-144) [ ] o.a.s.c.ZkTestServer
> Watch limit violations:
> [junit4] 2> Maximum concurrent create/delete watches above limit:
> [junit4] 2>
> [junit4] 2> 100 /solr/aliases.json
> [junit4] 2> 11 /solr/configs/conf
> [junit4] 2> 7 /solr/configs/ml
> [junit4] 2> 4 /solr/security.json
> [junit4] 2>
> [junit4] 2> Maximum concurrent data watches above limit:
> [junit4] 2>
> [junit4] 2> 100 /solr/clusterprops.json
> [junit4] 2> 100 /solr/clusterstate.json
> [junit4] 2> 38 /solr/collections/
> destinationCollection/state.json
> [junit4] 2> 27 /solr/collections/
> parallelDestinationCollection1/state.json
> [junit4] 2> 20 /solr/collections/
> parallelDestinationCollection/state.json
> [junit4] 2> 10 /solr/collections/mainCorpus1/state.json
> [junit4] 2> 10 /solr/collections/destination/state.json
> [junit4] 2> 10 /solr/collections/destination1/state.json
> [junit4] 2> 10 /solr/collections/collection1/state.json
> [junit4] 2> 10 /solr/collections/workQueue1/state.json
> [junit4] 2> 10 /solr/collections/workQueue/state.json
> [junit4] 2> 10 /solr/collections/
> uknownCollection/state.json
> [junit4] 2> 10 /solr/collections/
> modelCollection/state.json
> [junit4] 2> 10 /solr/collections/
> checkpointCollection/state.json
> [junit4] 2> 10 /solr/collections/mainCorpus/state.json
> [junit4] 2> 2 /solr/overseer_elect/election/
> 99400986694778891-127.0.0.1:34381_solr-n_0000000001
> [junit4] 2>
> [junit4] 2> Maximum concurrent children watches above limit:
> [junit4] 2>
> [junit4] 2> 100 /solr/live_nodes
> [junit4] 2> 100 /solr/collections
> [junit4] 2>
> [junit4] 2> NOTE: leaving temporary files on disk at:
> /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/
> build/solr-solrj/test/J2/temp/solr.client.solrj.io.stream.
> StreamExpressionTest_6471768B25648F68-001
> [junit4] 2> NOTE: test params are: codec=
> HighCompressionCompressingStoredFields(storedFieldsFormat=
> CompressingStoredFieldsFormat(compressionMode=HIGH_COMPRESSION,
> chunkSize=6, maxDocsPerChunk=6, blockSize=855), termVectorsFormat=
> CompressingTermVectorsFormat(compressionMode=HIGH_COMPRESSION,
> chunkSize=6, blockSize=855)), sim=RandomSimilarity(queryNorm=false): {},
> locale=es-VE, timezone=Atlantic/Faeroe
> [junit4] 2> NOTE: Linux 4.13.0-31-generic amd64/Oracle Corporation
> 10-ea (64-bit)/cpus=8,threads=1,free=140994872,total=525336576
> [junit4] 2> NOTE: All tests run in this JVM: [TangentEvaluatorTest,
> SolrExampleXMLTest, HyperbolicSineEvaluatorTest, NamedListTest,
> UuidEvaluatorTest, TestCloudSolrClientConnections, CommonParamsTest,
> TestHash, CloudSolrClientBuilderTest, DocumentAnalysisResponseTest,
> LBHttpSolrClientBuilderTest, SineEvaluatorTest, SolrExampleJettyTest,
> ArrayEvaluatorTest, LessThanEqualToEvaluatorTest, OperationsTest,
> CloudSolrClientTest, TestCollectionAdminRequest, CoalesceEvaluatorTest,
> CosineEvaluatorTest, ExclusiveOrEvaluatorTest, SelectWithEvaluatorsTest,
> TestCoreAdmin, SolrExceptionTest, HttpSolrClientBuilderTest,
> JdbcDriverTest, StreamExpressionTest]
> [junit4] Completed [146/146 (1!)] on J2 in 120.23s, 152 tests, 1
> failure <<< FAILURES!
>
> [...truncated 39995 lines...]
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org
>