You are viewing a plain text version of this content. The canonical link for it is here.
Posted to builds@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2020/02/07 03:27:01 UTC
[JENKINS] Lucene-Solr-master-Windows (64bit/jdk-11.0.6) - Build #
8434 - Still Unstable!
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Windows/8434/
Java: 64bit/jdk-11.0.6 -XX:+UseCompressedOops -XX:+UseG1GC
1 tests failed.
FAILED: org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration.testTriggerThrottling
Error Message:
Both triggers did not fire event after await()ing an excessive amount of time
Stack Trace:
java.lang.AssertionError: Both triggers did not fire event after await()ing an excessive amount of time
at __randomizedtesting.SeedInfo.seed([E1981D924104557D:1ABAB5B793AEB6EF]:0)
at org.junit.Assert.fail(Assert.java:88)
at org.junit.Assert.assertTrue(Assert.java:41)
at org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration.testTriggerThrottling(TestSimTriggerIntegration.java:197)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:566)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
at java.base/java.lang.Thread.run(Thread.java:834)
Build Log:
[...truncated 14161 lines...]
[junit4] Suite: org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration
[junit4] 2> 937293 INFO (SUITE-TestSimTriggerIntegration-seed#[E1981D924104557D]-worker) [ ] o.a.s.SolrTestCaseJ4 Created dataDir: C:\Users\jenkins\workspace\Lucene-Solr-master-Windows\solr\build\solr-core\test\J2\temp\solr.cloud.autoscaling.sim.TestSimTriggerIntegration_E1981D924104557D-001\data-dir-78-001
[junit4] 2> 937293 WARN (SUITE-TestSimTriggerIntegration-seed#[E1981D924104557D]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=12 numCloses=12
[junit4] 2> 937293 INFO (SUITE-TestSimTriggerIntegration-seed#[E1981D924104557D]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 937302 INFO (SUITE-TestSimTriggerIntegration-seed#[E1981D924104557D]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason="", value=0.0/0.0, ssl=0.0/0.0, clientAuth=0.0/0.0)
[junit4] 2> 937302 INFO (SUITE-TestSimTriggerIntegration-seed#[E1981D924104557D]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 937306 INFO (TEST-TestSimTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[E1981D924104557D]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testNodeAddedTriggerRestoreState
[junit4] 2> 937422 DEBUG (TEST-TestSimTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[E1981D924104557D]) [ ] o.a.s.c.a.s.SimClusterStateProvider --- new Overseer leader: 127.0.0.1:10520_solr
[junit4] 2> 937422 DEBUG (TEST-TestSimTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[E1981D924104557D]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=0
[junit4] 2> 937422 DEBUG (TEST-TestSimTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[E1981D924104557D]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 0
[junit4] 2> 937422 INFO (TEST-TestSimTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[E1981D924104557D]) [ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 937425 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
[junit4] 2> 937425 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
[junit4] 2> 937428 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 0
[junit4] 2> 937428 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
[junit4] 2> 937428 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10520_solr, 127.0.0.1:10521_solr]
[junit4] 2> 937428 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- deactivating old nodeLost / nodeAdded markers
[junit4] 2> 937428 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
[junit4] 2> 937430 DEBUG (ScheduledTrigger-4409-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 937430 DEBUG (simCloudManagerPool-4408-thread-2) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 937430 DEBUG (simCloudManagerPool-4408-thread-2) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2
[junit4] 2> 937436 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
[junit4] 2> 937444 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- deactivating old nodeLost / nodeAdded markers
[junit4] 2> 937444 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
[junit4] 2> 937459 DEBUG (ScheduledTrigger-4409-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 937459 DEBUG (simCloudManagerPool-4408-thread-3) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 937459 DEBUG (simCloudManagerPool-4408-thread-3) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 3
[junit4] 2> 937459 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
[junit4] 2> 937459 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestSimTriggerIntegration TestTriggerAction init
[junit4] 2> 937459 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger - Initial livenodes: [127.0.0.1:10520_solr, 127.0.0.1:10521_solr]
[junit4] 2> 937459 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger instantiated with properties: {event=nodeAdded, waitFor=500000, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration$TestTriggerAction}], enabled=true}
[junit4] 2> 937459 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- deactivating old nodeLost / nodeAdded markers
[junit4] 2> 937459 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
[junit4] 2> 937459 DEBUG (ScheduledTrigger-4409-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger
[junit4] 2> 937459 DEBUG (ScheduledTrigger-4409-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
[junit4] 2> 937467 DEBUG (TEST-TestSimTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[E1981D924104557D]) [ ] o.a.s.c.a.s.SimClusterStateProvider -- creating marker: /autoscaling/nodeAdded/127.0.0.1:10522_solr
[junit4] 2> 937492 DEBUG (ScheduledTrigger-4409-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 2
[junit4] 2> 937512 DEBUG (ScheduledTrigger-4409-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger
[junit4] 2> 937512 DEBUG (ScheduledTrigger-4409-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 937513 DEBUG (ScheduledTrigger-4409-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10522_solr at time 703347112327000
[junit4] 2> 937528 DEBUG (simCloudManagerPool-4408-thread-4) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 937528 DEBUG (simCloudManagerPool-4408-thread-4) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 4
[junit4] 2> 937528 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 4
[junit4] 2> 937528 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestSimTriggerIntegration TestTriggerAction init
[junit4] 2> 937528 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger - Initial livenodes: [127.0.0.1:10520_solr, 127.0.0.1:10521_solr, 127.0.0.1:10522_solr]
[junit4] 2> 937528 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration$TestTriggerAction}], enabled=true}
[junit4] 2> 937528 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- deactivating old nodeLost / nodeAdded markers
[junit4] 2> 937528 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion 4
[junit4] 2> 937544 DEBUG (ScheduledTrigger-4409-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 937544 DEBUG (ScheduledTrigger-4409-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger
[junit4] 2> 937544 DEBUG (ScheduledTrigger-4409-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 937544 DEBUG (ScheduledTrigger-4409-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger firing registered processor for nodes: [127.0.0.1:10522_solr] added at times [703347112327000], now=703348717117000
[junit4] 2> 937544 DEBUG (ScheduledTrigger-4409-thread-1) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
[junit4] 2> "id":"27fb0c2c8b758T9lctjnsnympiiyillrhvo70ea",
[junit4] 2> "source":"node_added_restore_trigger",
[junit4] 2> "eventTime":703347112327000,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[703347112327000],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "nodeNames":["127.0.0.1:10522_solr"],
[junit4] 2> "replicaType":"NRT"}}
[junit4] 2> 937544 DEBUG (ScheduledTrigger-4409-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_added_restore_trigger]
[junit4] 2> 937544 DEBUG (ScheduledTrigger-4409-thread-1) [ ] o.a.s.c.a.SystemLogListener Collection .system missing, skip sending event {
[junit4] 2> "id":"27fb0c2c8b758T9lctjnsnympiiyillrhvo70ea",
[junit4] 2> "source":"node_added_restore_trigger",
[junit4] 2> "eventTime":703347112327000,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[703347112327000],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":703348741147000,
[junit4] 2> "nodeNames":["127.0.0.1:10522_solr"],
[junit4] 2> "replicaType":"NRT"}}
[junit4] 2> 937544 DEBUG (ScheduledTrigger-4409-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Removing new node from tracking: 127.0.0.1:10522_solr
[junit4] 2> 937544 DEBUG (AutoscalingActionExecutor-4410-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
[junit4] 2> "id":"27fb0c2c8b758T9lctjnsnympiiyillrhvo70ea",
[junit4] 2> "source":"node_added_restore_trigger",
[junit4] 2> "eventTime":703347112327000,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[703347112327000],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":703348741147000,
[junit4] 2> "nodeNames":["127.0.0.1:10522_solr"],
[junit4] 2> "replicaType":"NRT"}}
[junit4] 2> 937560 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 937560 DEBUG (TEST-TestSimTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[E1981D924104557D]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
[junit4] 2> 937560 WARN (AutoscalingActionExecutor-4410-thread-1) [ ] o.a.s.c.a.SystemLogListener Exception sending event. Collection: .system, event: {
[junit4] 2> "id":"27fb0c2c8b758T9lctjnsnympiiyillrhvo70ea",
[junit4] 2> "source":"node_added_restore_trigger",
[junit4] 2> "eventTime":703347112327000,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[703347112327000],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":703348741147000,
[junit4] 2> "nodeNames":["127.0.0.1:10522_solr"],
[junit4] 2> "replicaType":"NRT"}}, exception: java.io.IOException: already closed
[junit4] 2> 937560 DEBUG (AutoscalingActionExecutor-4410-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 1296 ms for event id=27fb0c2c8b758T9lctjnsnympiiyillrhvo70ea
[junit4] 2> 937579 DEBUG (TEST-TestSimTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[E1981D924104557D]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now
[junit4] 2> 937579 DEBUG (TEST-TestSimTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[E1981D924104557D]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
[junit4] 2> 937579 DEBUG (TEST-TestSimTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[E1981D924104557D]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor
[junit4] 2> 937580 DEBUG (TEST-TestSimTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[E1981D924104557D]) [ ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
[junit4] 2> 937580 DEBUG (TEST-TestSimTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[E1981D924104557D]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
[junit4] 2> 937580 INFO (TEST-TestSimTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[E1981D924104557D]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testNodeAddedTriggerRestoreState
[junit4] 2> 937591 INFO (TEST-TestSimTriggerIntegration.testTriggerThrottling-seed#[E1981D924104557D]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testTriggerThrottling
[junit4] 2> 937724 DEBUG (TEST-TestSimTriggerIntegration.testTriggerThrottling-seed#[E1981D924104557D]) [ ] o.a.s.c.a.s.SimClusterStateProvider --- new Overseer leader: 127.0.0.1:10523_solr
[junit4] 2> 937725 DEBUG (TEST-TestSimTriggerIntegration.testTriggerThrottling-seed#[E1981D924104557D]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=0
[junit4] 2> 937725 DEBUG (TEST-TestSimTriggerIntegration.testTriggerThrottling-seed#[E1981D924104557D]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 0
[junit4] 2> 937725 INFO (TEST-TestSimTriggerIntegration.testTriggerThrottling-seed#[E1981D924104557D]) [ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 937725 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
[junit4] 2> 937728 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
[junit4] 2> 937728 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 0
[junit4] 2> 937728 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
[junit4] 2> 937728 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10524_solr, 127.0.0.1:10523_solr]
[junit4] 2> 937728 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- deactivating old nodeLost / nodeAdded markers
[junit4] 2> 937728 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
[junit4] 2> 937728 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 937728 DEBUG (simCloudManagerPool-4414-thread-3) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 937728 DEBUG (simCloudManagerPool-4414-thread-3) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2
[junit4] 2> 937752 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
[junit4] 2> 937752 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- deactivating old nodeLost / nodeAdded markers
[junit4] 2> 937752 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
[junit4] 2> 937756 DEBUG (simCloudManagerPool-4414-thread-4) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 937756 DEBUG (simCloudManagerPool-4414-thread-4) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 3
[junit4] 2> 937756 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
[junit4] 2> 937756 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestSimTriggerIntegration TestTriggerAction init
[junit4] 2> 937756 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 - Initial livenodes: [127.0.0.1:10524_solr, 127.0.0.1:10523_solr]
[junit4] 2> 937756 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 instantiated with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration$ThrottlingTesterAction}], enabled=true}
[junit4] 2> 937756 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- deactivating old nodeLost / nodeAdded markers
[junit4] 2> 937756 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
[junit4] 2> 937786 DEBUG (simCloudManagerPool-4414-thread-5) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 937786 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 937786 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 937786 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
[junit4] 2> 937786 DEBUG (simCloudManagerPool-4414-thread-5) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 4
[junit4] 2> 937786 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 4
[junit4] 2> 937786 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestSimTriggerIntegration TestTriggerAction init
[junit4] 2> 937786 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger2 - Initial livenodes: [127.0.0.1:10524_solr, 127.0.0.1:10523_solr]
[junit4] 2> 937786 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger2 instantiated with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration$ThrottlingTesterAction}], enabled=true}
[junit4] 2> 937786 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- deactivating old nodeLost / nodeAdded markers
[junit4] 2> 937786 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion 4
[junit4] 2> 937786 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 937786 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
[junit4] 2> 937800 DEBUG (TEST-TestSimTriggerIntegration.testTriggerThrottling-seed#[E1981D924104557D]) [ ] o.a.s.c.a.s.SimClusterStateProvider -- creating marker: /autoscaling/nodeAdded/127.0.0.1:10525_solr
[junit4] 2> 937824 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 2
[junit4] 2> 937827 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 937827 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 937827 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10525_solr at time 703362918642000
[junit4] 2> 937827 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger2 firing registered processor for nodes: [127.0.0.1:10525_solr] added at times [703362918642000], now=703362920687000
[junit4] 2> 937827 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 937827 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 937827 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10525_solr at time 703362931222000
[junit4] 2> 937827 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 firing registered processor for nodes: [127.0.0.1:10525_solr] added at times [703362931222000], now=703362931762000
[junit4] 2> 937827 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
[junit4] 2> "id":"27fb471a9e5f0T9lctjnsnympiiyillrhvo70ed",
[junit4] 2> "source":"node_added_trigger1",
[junit4] 2> "eventTime":703362931222000,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[703362931222000],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "nodeNames":["127.0.0.1:10525_solr"],
[junit4] 2> "replicaType":"NRT"}}
[junit4] 2> 937856 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_added_trigger2, node_added_trigger1]
[junit4] 2> 937856 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.SystemLogListener Collection .system missing, skip sending event {
[junit4] 2> "id":"27fb471a9e5f0T9lctjnsnympiiyillrhvo70ed",
[junit4] 2> "source":"node_added_trigger1",
[junit4] 2> "eventTime":703362931222000,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[703362931222000],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":703364308482000,
[junit4] 2> "nodeNames":["127.0.0.1:10525_solr"],
[junit4] 2> "replicaType":"NRT"}}
[junit4] 2> 937856 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Removing new node from tracking: 127.0.0.1:10525_solr
[junit4] 2> 937856 DEBUG (AutoscalingActionExecutor-4416-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
[junit4] 2> "id":"27fb471a9e5f0T9lctjnsnympiiyillrhvo70ed",
[junit4] 2> "source":"node_added_trigger1",
[junit4] 2> "eventTime":703362931222000,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[703362931222000],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":703364308482000,
[junit4] 2> "nodeNames":["127.0.0.1:10525_solr"],
[junit4] 2> "replicaType":"NRT"}}
[junit4] 2> 937856 INFO (AutoscalingActionExecutor-4416-thread-1) [ ] o.a.s.c.a.s.TestSimTriggerIntegration action executed from node_added_trigger1
[junit4] 2> 937856 DEBUG (AutoscalingActionExecutor-4416-thread-1) [ ] o.a.s.c.a.SystemLogListener Collection .system missing, skip sending event {
[junit4] 2> "id":"27fb471a9e5f0T9lctjnsnympiiyillrhvo70ed",
[junit4] 2> "source":"node_added_trigger1",
[junit4] 2> "eventTime":703362931222000,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[703362931222000],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":703364308482000,
[junit4] 2> "nodeNames":["127.0.0.1:10525_solr"],
[junit4] 2> "replicaType":"NRT"}}
[junit4] 2> 937856 DEBUG (AutoscalingActionExecutor-4416-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger2 after 100ms
[junit4] 2> 937856 DEBUG (AutoscalingActionExecutor-4416-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms
[junit4] 2> 937856 DEBUG (AutoscalingActionExecutor-4416-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger1 after 100ms
[junit4] 2> 937856 DEBUG (AutoscalingActionExecutor-4416-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 30 ms for event id=27fb471a9e5f0T9lctjnsnympiiyillrhvo70ed
[junit4] 2> 937861 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
[junit4] 2> "id":"27fb470e9f150T9lctjnsnympiiyillrhvo70ec",
[junit4] 2> "source":"node_added_trigger2",
[junit4] 2> "eventTime":703362918642000,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[703362918642000],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "nodeNames":["127.0.0.1:10525_solr"],
[junit4] 2> "replicaType":"NRT"}}
[junit4] 2> 937861 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_added_trigger2, node_added_trigger1]
[junit4] 2> 937861 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.SystemLogListener Collection .system missing, skip sending event {
[junit4] 2> "id":"27fb470e9f150T9lctjnsnympiiyillrhvo70ec",
[junit4] 2> "source":"node_added_trigger2",
[junit4] 2> "eventTime":703362918642000,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[703362918642000],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":703364553297000,
[junit4] 2> "nodeNames":["127.0.0.1:10525_solr"],
[junit4] 2> "replicaType":"NRT"}}
[junit4] 2> 937861 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Removing new node from tracking: 127.0.0.1:10525_solr
[junit4] 2> 937862 DEBUG (AutoscalingActionExecutor-4416-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
[junit4] 2> "id":"27fb470e9f150T9lctjnsnympiiyillrhvo70ec",
[junit4] 2> "source":"node_added_trigger2",
[junit4] 2> "eventTime":703362918642000,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[703362918642000],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":703364553297000,
[junit4] 2> "nodeNames":["127.0.0.1:10525_solr"],
[junit4] 2> "replicaType":"NRT"}}
[junit4] 2> 937866 INFO (AutoscalingActionExecutor-4416-thread-1) [ ] o.a.s.c.a.s.TestSimTriggerIntegration last action at 703364379977000 time = 703364600722000
[junit4] 2> 937866 INFO (AutoscalingActionExecutor-4416-thread-1) [ ] o.a.s.c.a.s.TestSimTriggerIntegration action executed again before minimum wait time from node_added_trigger2
[junit4] 2> 937866 DEBUG (AutoscalingActionExecutor-4416-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger1 after 100ms
[junit4] 2> 937866 DEBUG (AutoscalingActionExecutor-4416-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger2 after 100ms
[junit4] 2> 937866 DEBUG (AutoscalingActionExecutor-4416-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms
[junit4] 2> 937968 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 937968 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 937968 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 937968 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 937968 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 937992 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 937992 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 937992 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 937992 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 937992 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938012 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 938012 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938012 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 938012 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938012 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 938033 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 938033 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 938033 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938033 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938033 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 938053 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 938053 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 938053 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938053 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 938053 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938076 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 938076 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 938076 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938076 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 938076 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938100 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 938100 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 938100 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938100 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 938100 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938122 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 938122 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 938122 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 938122 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938122 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938152 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 938152 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 938152 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938152 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 938153 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938172 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 938177 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 938177 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938177 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 938177 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938204 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 938204 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938204 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 938204 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938204 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 938226 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 938226 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938226 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 938226 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 938226 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938252 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 938252 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938252 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 938252 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 938252 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938274 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 938274 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938274 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 938274 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 938274 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938294 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 938294 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938294 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 938294 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 938294 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938316 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 938316 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 938316 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938316 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 938316 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938341 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 938341 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 938341 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938341 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 938341 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938363 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 938363 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 938363 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938363 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 938363 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938388 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 938388 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938388 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 938388 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 938388 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938409 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 938409 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 938409 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938409 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 938409 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938432 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 938432 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 938432 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938432 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 938432 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938455 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 938455 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 938455 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938455 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 938455 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938477 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 938477 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 938477 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938477 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 938477 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938498 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 938498 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 938498 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938498 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 938498 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938519 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 938519 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 938519 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938519 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 938519 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938540 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 938540 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938540 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 938540 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938541 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 938564 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 938564 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938564 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 938564 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938564 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 938588 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 938588 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938588 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 938588 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938589 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 938614 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 938614 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938614 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 938614 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938614 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 938636 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 938636 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938636 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 938636 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938636 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 938660 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 938660 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 938660 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938660 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 938660 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938684 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 938684 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938684 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 938684 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 938684 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938707 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 938707 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938707 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 938707 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 938707 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938730 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 938730 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938730 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 938730 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 938730 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938750 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 938750 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938750 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 938750 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 938750 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938772 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 938772 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938772 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 938772 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 938772 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938794 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 938794 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 938794 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938794 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 938794 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938817 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 938817 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 938817 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938817 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 938817 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938841 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 938841 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938841 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 938841 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 938841 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938861 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 938861 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 938861 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938861 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 938861 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938884 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 938884 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938884 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 938884 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 938884 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938904 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 938904 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938904 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 938904 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 938904 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938924 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 938924 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938924 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 938924 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938924 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 938946 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 938946 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938946 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 938946 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 938946 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938968 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 938968 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938968 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 938968 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 938968 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938990 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 938990 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 938990 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 938990 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 938990 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939012 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 939012 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939012 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 939012 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 939012 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939033 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 939033 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939033 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 939033 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 939033 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939055 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 939055 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939055 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 939055 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 939055 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939077 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 939077 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939077 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 939077 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 939077 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939100 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 939100 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 939100 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939100 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939100 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 939121 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 939121 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939121 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 939121 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939121 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 939144 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 939144 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 939144 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 939144 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939144 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939171 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 939171 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 939171 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939171 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 939171 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939194 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 939194 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 939194 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939194 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 939194 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939216 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 939216 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 939216 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939216 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 939216 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939241 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 939241 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 939241 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939241 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 939241 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939272 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 939272 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 939272 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939272 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 939272 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939293 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 939293 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 939293 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939293 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 939293 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939317 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 939317 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 939317 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939317 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939317 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 939339 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 939339 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939339 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 939339 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 939339 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939362 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 939362 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939362 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 939362 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 939362 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939395 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 939395 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 939395 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939395 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 939395 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939420 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 939420 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 939420 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939420 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 939420 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939440 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 939440 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 939440 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939440 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 939440 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939476 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 939476 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 939476 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 939476 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939476 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939501 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 939501 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 939501 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939501 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 939501 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939525 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 939525 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 939525 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939525 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 939525 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939548 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 939548 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939548 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 939548 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939548 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 939572 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 939572 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939572 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 939572 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939572 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 939596 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 939596 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 939596 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939596 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939596 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 939617 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 939617 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 939617 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939617 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 939617 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939640 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 939640 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 939640 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939640 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939640 DEBUG (ScheduledTrigger-4415-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 939661 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 939661 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 939661 DEBUG (ScheduledTrigger-4415-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939661 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939661 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 939685 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 939685 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939685 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 939685 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939685 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 939706 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 939706 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939706 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 939706 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 939706 DEBUG (ScheduledTrigger-4415-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939728 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 939728 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939728 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 939728 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 939728 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 939750 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 939750 DEBUG (ScheduledTrigger-4415-thread-1) [ ] o.a.s.c.a.
[...truncated too long message...]
> 1008475 DEBUG (ScheduledTrigger-4480-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 and last live nodes: 4
[junit4] 2> 1008502 DEBUG (ScheduledTrigger-4480-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 1008502 DEBUG (ScheduledTrigger-4480-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 1008502 DEBUG (ScheduledTrigger-4480-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 and last live nodes: 4
[junit4] 2> 1008534 DEBUG (ScheduledTrigger-4480-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 1008534 DEBUG (ScheduledTrigger-4480-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 1008534 DEBUG (ScheduledTrigger-4480-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger firing registered processor for nodes: [127.0.0.1:10554_solr] added at times [706887765177000], now=706898556437000
[junit4] 2> 1008542 DEBUG (ScheduledTrigger-4480-thread-4) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
[junit4] 2> "id":"282e92223b2a8T9lctjnsnympiiyillrhvo70f1",
[junit4] 2> "source":"node_added_trigger",
[junit4] 2> "eventTime":706887765177000,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[706887765177000],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "nodeNames":["127.0.0.1:10554_solr"],
[junit4] 2> "replicaType":"NRT"}}
[junit4] 2> 1008543 DEBUG (ScheduledTrigger-4480-thread-4) [ ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_added_trigger]
[junit4] 2> 1008543 DEBUG (ScheduledTrigger-4480-thread-4) [ ] o.a.s.c.a.SystemLogListener Collection .system missing, skip sending event {
[junit4] 2> "id":"282e92223b2a8T9lctjnsnympiiyillrhvo70f1",
[junit4] 2> "source":"node_added_trigger",
[junit4] 2> "eventTime":706887765177000,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[706887765177000],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":706898821507000,
[junit4] 2> "nodeNames":["127.0.0.1:10554_solr"],
[junit4] 2> "replicaType":"NRT"}}
[junit4] 2> 1008543 DEBUG (ScheduledTrigger-4480-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Removing new node from tracking: 127.0.0.1:10554_solr
[junit4] 2> 1008554 DEBUG (AutoscalingActionExecutor-4481-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
[junit4] 2> "id":"282e92223b2a8T9lctjnsnympiiyillrhvo70f1",
[junit4] 2> "source":"node_added_trigger",
[junit4] 2> "eventTime":706887765177000,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[706887765177000],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":706898821507000,
[junit4] 2> "nodeNames":["127.0.0.1:10554_solr"],
[junit4] 2> "replicaType":"NRT"}}
[junit4] 2> 1008554 DEBUG (AutoscalingActionExecutor-4481-thread-1) [ ] o.a.s.c.a.SystemLogListener Collection .system missing, skip sending event {
[junit4] 2> "id":"282e92223b2a8T9lctjnsnympiiyillrhvo70f1",
[junit4] 2> "source":"node_added_trigger",
[junit4] 2> "eventTime":706887765177000,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[706887765177000],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":706898821507000,
[junit4] 2> "nodeNames":["127.0.0.1:10554_solr"],
[junit4] 2> "replicaType":"NRT"}}
[junit4] 2> 1008554 DEBUG (AutoscalingActionExecutor-4481-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger after 100ms
[junit4] 2> 1008554 DEBUG (AutoscalingActionExecutor-4481-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms
[junit4] 2> 1008554 DEBUG (AutoscalingActionExecutor-4481-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 478 ms for event id=282e92223b2a8T9lctjnsnympiiyillrhvo70f1
[junit4] 2> 1008562 DEBUG (TEST-TestSimTriggerIntegration.testEventFromRestoredState-seed#[E1981D924104557D]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
[junit4] 2> 1008562 DEBUG (TEST-TestSimTriggerIntegration.testEventFromRestoredState-seed#[E1981D924104557D]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now
[junit4] 2> 1008562 DEBUG (TEST-TestSimTriggerIntegration.testEventFromRestoredState-seed#[E1981D924104557D]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
[junit4] 2> 1008562 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 1008562 DEBUG (TEST-TestSimTriggerIntegration.testEventFromRestoredState-seed#[E1981D924104557D]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor
[junit4] 2> NOTE: leaving temporary files on disk at: C:\Users\jenkins\workspace\Lucene-Solr-master-Windows\solr\build\solr-core\test\J2\temp\solr.cloud.autoscaling.sim.TestSimTriggerIntegration_E1981D924104557D-001
[junit4] 2> 1008566 DEBUG (TEST-TestSimTriggerIntegration.testEventFromRestoredState-seed#[E1981D924104557D]) [ ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
[junit4] 2> 1008566 DEBUG (TEST-TestSimTriggerIntegration.testEventFromRestoredState-seed#[E1981D924104557D]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
[junit4] 2> 1008566 INFO (TEST-TestSimTriggerIntegration.testEventFromRestoredState-seed#[E1981D924104557D]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testEventFromRestoredState
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene84): {}, docValues:{}, maxPointsInLeafNode=967, maxMBSortInHeap=6.942519614489948, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@6e062ab2), locale=et-EE, timezone=Pacific/Kosrae
[junit4] 2> NOTE: Windows 10 10.0 amd64/AdoptOpenJDK 11.0.6 (64-bit)/cpus=6,threads=1,free=312144680,total=498073600
[junit4] 2> NOTE: All tests run in this JVM: [TriggerEventQueueTest, TestLegacyField, DirectUpdateHandlerTest, TestLuceneMatchVersion, SolrGraphiteReporterTest, TestStandardQParsers, TestStressInPlaceUpdates, TestSolrJacksonAnnotation, DocExpirationUpdateProcessorFactoryTest, StressHdfsTest, SolrCLIZkUtilsTest, TlogReplayBufferedWhileIndexingTest, ResponseBuilderTest, TestCrossCoreJoin, TestDocumentBuilder, TestRandomCollapseQParserPlugin, FullHLLTest, TestDownShardTolerantSearch, CollectionsAPIDistributedZkTest, MultiDestinationAuditLoggerTest, TestSimDistribStateManager, TestCustomSort, DocValuesTest, TestDocTermOrdsUninvertLimit, MoveReplicaHDFSFailoverTest, TestSuggestSpellingConverter, CdcrWithNodesRestartsTest, TestSlowCompositeReaderWrapper, TestJsonRequest, TestSolrConfigHandlerCloud, HighlighterConfigTest, TestTrie, TestSkipOverseerOperations, TestCollationFieldDocValues, ConfigSetsAPITest, FacetPivotSmallTest, TestSimLargeCluster, ImplicitSnitchTest, TestPullReplicaErrorHandling, TestFieldTypeResource, ChaosMonkeySafeLeaderWithPullReplicasTest, TestTolerantUpdateProcessorRandomCloud, TestOmitPositions, TestDynamicFieldCollectionResource, TestIndexSearcher, ClusterStateUpdateTest, TestSortableTextField, TestDFISimilarityFactory, TestLRUStatsCache, TestJsonFacetsWithNestedObjects, ZookeeperStatusHandlerTest, URLClassifyProcessorTest, TestLockTree, TestConfigReload, ResponseHeaderTest, MinimalSchemaTest, FileUtilsTest, TestStreamBody, DistributedQueryComponentOptimizationTest, BasicAuthStandaloneTest, TestEmbeddedSolrServerAdminHandler, TestExceedMaxTermLength, AutoAddReplicasPlanActionTest, Tagger2Test, TestSimpleQParserPlugin, DaemonStreamApiTest, TestRawResponseWriter, ReplicationFactorTest, ReturnFieldsTest, DocValuesNotIndexedTest, SystemCollectionCompatTest, TestSolrCoreParser, TestDynamicLoadingUrl, TestZkAclsWithHadoopAuth, TestSolrDeletionPolicy1, TestBinaryField, TestRandomFaceting, TestDocBasedVersionConstraints, CoreAdminCreateDiscoverTest, NestedAtomicUpdateTest, SolrShardReporterTest, SolrCoreTest, TestSimTriggerIntegration]
[junit4] Completed [404/900 (1!)] on J2 in 71.42s, 11 tests, 1 failure <<< FAILURES!
[...truncated 54271 lines...]
[repro] Jenkins log URL: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Windows/8434/consoleText
[repro] Revision: abd282d258d23d19b7f7c1e96332a19fa7b7b827
[repro] Ant options: "-Dargs=-XX:+UseCompressedOops -XX:+UseG1GC"
[repro] JUnit rest result XML files will be moved to: ./repro-reports
[repro] ant clean
[...truncated 6 lines...]
[repro] Test suites by module:
[repro] solr\core
[repro] TestSimTriggerIntegration
[repro] ant compile-test
[...truncated 3168 lines...]
[repro] ant test-nocompile -Dtests.dups=5 -Dtests.maxfailures=5 -Dtests.class="*.TestSimTriggerIntegration" -Dtests.showOutput=onerror "-Dargs=-XX:+UseCompressedOops -XX:+UseG1GC" -Dtests.seed=E1981D924104557D -Dtests.slow=true -Dtests.locale=et-EE -Dtests.timezone=Pacific/Kosrae -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
[...truncated 132 lines...]
[repro] Failures w/original seeds:
[repro] 0/5 failed: org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration
[repro] Exiting with code 0
[...truncated 78 lines...]