You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2018/09/13 01:21:50 UTC
[JENKINS] Lucene-Solr-Tests-master - Build # 2807 - Unstable
Build: https://builds.apache.org/job/Lucene-Solr-Tests-master/2807/
1 tests failed.
FAILED: org.apache.solr.cloud.autoscaling.sim.TestSimComputePlanAction.testNodeAdded
Error Message:
ComputePlanAction should have computed exactly 1 operation, but was: [org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@2c8dc325, org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@14e31142] expected:<1> but was:<2>
Stack Trace:
java.lang.AssertionError: ComputePlanAction should have computed exactly 1 operation, but was: [org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@2c8dc325, org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@14e31142] expected:<1> but was:<2>
at __randomizedtesting.SeedInfo.seed([69F3AA28CE054E1:635C6CD52E43FCE2]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:472)
at org.apache.solr.cloud.autoscaling.sim.TestSimComputePlanAction.testNodeAdded(TestSimComputePlanAction.java:314)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:935)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:971)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:985)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:944)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:830)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:880)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:891)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 12606 lines...]
[junit4] Suite: org.apache.solr.cloud.autoscaling.sim.TestSimComputePlanAction
[junit4] 2> 179603 INFO (SUITE-TestSimComputePlanAction-seed#[69F3AA28CE054E1]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.sim.TestSimComputePlanAction_69F3AA28CE054E1-001/init-core-data-001
[junit4] 2> 179604 WARN (SUITE-TestSimComputePlanAction-seed#[69F3AA28CE054E1]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=4 numCloses=4
[junit4] 2> 179604 INFO (SUITE-TestSimComputePlanAction-seed#[69F3AA28CE054E1]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 179607 INFO (SUITE-TestSimComputePlanAction-seed#[69F3AA28CE054E1]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 179682 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
[junit4] 2> 179683 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 0
[junit4] 2> 179684 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion -1
[junit4] 2> 179684 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 0
[junit4] 2> 179685 DEBUG (SUITE-TestSimComputePlanAction-seed#[69F3AA28CE054E1]-worker) [ ] o.a.s.c.a.s.SimClusterStateProvider --- new Overseer leader: 127.0.0.1:10000_solr
[junit4] 2> 179703 INFO (SUITE-TestSimComputePlanAction-seed#[69F3AA28CE054E1]-worker) [ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 179717 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10000_solr]
[junit4] 2> 179718 INFO (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[69F3AA28CE054E1]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testNodeWithMultipleReplicasLost
[junit4] 2> 179718 DEBUG (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
[junit4] 2> 179719 INFO (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
[junit4] 2> 179724 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 179724 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 0
[junit4] 2> 179724 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
[junit4] 2> 179724 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
[junit4] 2> 179724 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 179724 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
[junit4] 2> 179725 DEBUG (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
[junit4] 2> 179725 DEBUG (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now
[junit4] 2> 179725 DEBUG (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
[junit4] 2> 179725 DEBUG (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor
[junit4] 2> 179725 DEBUG (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
[junit4] 2> 179725 DEBUG (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
[junit4] 2> 179725 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 179740 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
[junit4] 2> 179741 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2
[junit4] 2> 179745 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion -1
[junit4] 2> 179745 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
[junit4] 2> 179757 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10000_solr]
[junit4] 2> 179765 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 179765 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
[junit4] 2> 179765 DEBUG (ScheduledTrigger-885-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 179808 DEBUG (ScheduledTrigger-885-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 180059 DEBUG (ScheduledTrigger-885-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 180079 DEBUG (ScheduledTrigger-885-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 180097 DEBUG (simCloudManagerPool-884-thread-1) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 180098 DEBUG (simCloudManagerPool-884-thread-1) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 3
[junit4] 2> 180099 DEBUG (ScheduledTrigger-885-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 180104 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
[junit4] 2> 180104 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 180104 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
[junit4] 2> 180118 DEBUG (simCloudManagerPool-884-thread-2) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 180119 DEBUG (ScheduledTrigger-885-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 180120 DEBUG (simCloudManagerPool-884-thread-2) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 4
[junit4] 2> 180125 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 4
[junit4] 2> 180125 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 180125 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion 4
[junit4] 2> 180140 DEBUG (ScheduledTrigger-885-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 180160 DEBUG (ScheduledTrigger-885-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 180180 DEBUG (ScheduledTrigger-885-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 180201 DEBUG (ScheduledTrigger-885-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 180221 DEBUG (ScheduledTrigger-885-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 180241 DEBUG (ScheduledTrigger-885-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
[junit4] 2> 180262 DEBUG (ScheduledTrigger-885-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
[junit4] 2> 180277 DEBUG (simCloudManagerPool-884-thread-3) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 180278 DEBUG (simCloudManagerPool-884-thread-3) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 5
[junit4] 2> 180283 DEBUG (ScheduledTrigger-885-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
[junit4] 2> 180285 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 5
[junit4] 2> 180285 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger node_lost_trigger - Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10000_solr, 127.0.0.1:10002_solr, 127.0.0.1:10003_solr]
[junit4] 2> 180285 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 180285 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 5, lastZnodeVersion 5
[junit4] 2> 180285 DEBUG (ScheduledTrigger-885-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 4
[junit4] 2> 180304 DEBUG (ScheduledTrigger-885-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
[junit4] 2> 180305 DEBUG (simCloudManagerPool-884-thread-4) [ ] o.a.s.c.o.ClusterStateMutator building a new cName: testNodeWithMultipleReplicasLost
[junit4] 2> 180307 DEBUG (ScheduledTrigger-885-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 4
[junit4] 2> 180324 DEBUG (ScheduledTrigger-885-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
[junit4] 2> 180328 DEBUG (ScheduledTrigger-885-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 4
[junit4] 2> 180343 INFO (simCloudManagerPool-884-thread-4) [ ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 4086766644 sessionWrapper.createTime 4086765668681206, this.sessionWrapper.createTime 4086765668681206
[junit4] 2> 180344 DEBUG (ScheduledTrigger-885-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
[junit4] 2> 180348 DEBUG (ScheduledTrigger-885-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 4
[junit4] 2> 180364 DEBUG (ScheduledTrigger-885-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
[junit4] 2> 180368 DEBUG (ScheduledTrigger-885-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 4
[junit4] 2> 180384 DEBUG (ScheduledTrigger-885-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
[junit4] 2> 180388 DEBUG (ScheduledTrigger-885-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 4
[junit4] 2> 180398 DEBUG (simCloudManagerPool-884-thread-5) [ ] o.a.s.c.a.s.SimClusterStateProvider -- submit leader election for testNodeWithMultipleReplicasLost / shard1
[junit4] 2> 180404 DEBUG (ScheduledTrigger-885-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
[junit4] 2> 180408 DEBUG (ScheduledTrigger-885-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 4
[junit4] 2> 180410 DEBUG (simCloudManagerPool-884-thread-9) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election for testNodeWithMultipleReplicasLost / shard1
[junit4] 2> 180419 DEBUG (simCloudManagerPool-884-thread-9) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeWithMultipleReplicasLost / shard1: core_node1
[junit4] 2> 180421 DEBUG (simCloudManagerPool-884-thread-6) [ ] o.a.s.c.a.s.SimClusterStateProvider -- already has leader testNodeWithMultipleReplicasLost / shard1
[junit4] 2> 180425 DEBUG (ScheduledTrigger-885-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
[junit4] 2> 180428 DEBUG (ScheduledTrigger-885-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 4
[junit4] 2> 180437 DEBUG (simCloudManagerPool-884-thread-7) [ ] o.a.s.c.a.s.SimClusterStateProvider -- no replicas in testNodeWithMultipleReplicasLost / shard2
[junit4] 2> 180437 DEBUG (simCloudManagerPool-884-thread-7) [ ] o.a.s.c.a.s.SimClusterStateProvider -- already has leader testNodeWithMultipleReplicasLost / shard1
[junit4] 2> 180445 DEBUG (ScheduledTrigger-885-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
[junit4] 2> 180449 DEBUG (simCloudManagerPool-884-thread-8) [ ] o.a.s.c.a.s.SimClusterStateProvider -- submit leader election for testNodeWithMultipleReplicasLost / shard2
[junit4] 2> 180455 DEBUG (simCloudManagerPool-884-thread-8) [ ] o.a.s.c.a.s.SimClusterStateProvider -- already has leader testNodeWithMultipleReplicasLost / shard1
[junit4] 2> 180455 DEBUG (simCloudManagerPool-884-thread-12) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election for testNodeWithMultipleReplicasLost / shard2
[junit4] 2> 180456 DEBUG (ScheduledTrigger-885-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 4
[junit4] 2> 180456 DEBUG (simCloudManagerPool-884-thread-12) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeWithMultipleReplicasLost / shard2: core_node4
[junit4] 2> 180466 DEBUG (ScheduledTrigger-885-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
[junit4] 2> 180466 DEBUG (simCloudManagerPool-884-thread-10) [ ] o.a.s.c.a.s.SimClusterStateProvider -- already has leader testNodeWithMultipleReplicasLost / shard2
[junit4] 2> 180466 DEBUG (simCloudManagerPool-884-thread-10) [ ] o.a.s.c.a.s.SimClusterStateProvider -- already has leader testNodeWithMultipleReplicasLost / shard1
[junit4] 2> 180476 DEBUG (ScheduledTrigger-885-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 4
[junit4] 2> 180477 DEBUG (simCloudManagerPool-884-thread-11) [ ] o.a.s.c.a.s.SimClusterStateProvider -- already has leader testNodeWithMultipleReplicasLost / shard2
[junit4] 2> 180477 DEBUG (simCloudManagerPool-884-thread-11) [ ] o.a.s.c.a.s.SimClusterStateProvider -- already has leader testNodeWithMultipleReplicasLost / shard1
[junit4] 2> 180482 DEBUG (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.s.TestSimComputePlanAction -- cluster state: znodeVersion: 10
[junit4] 2> live nodes:[127.0.0.1:10001_solr, 127.0.0.1:10000_solr, 127.0.0.1:10002_solr, 127.0.0.1:10003_solr]
[junit4] 2> collections:{testNodeWithMultipleReplicasLost=DocCollection(testNodeWithMultipleReplicasLost//clusterstate.json/9)={
[junit4] 2> "replicationFactor":"1",
[junit4] 2> "pullReplicas":"0",
[junit4] 2> "router":{"name":"compositeId"},
[junit4] 2> "maxShardsPerNode":"1",
[junit4] 2> "autoAddReplicas":"false",
[junit4] 2> "nrtReplicas":"3",
[junit4] 2> "tlogReplicas":"0",
[junit4] 2> "autoCreated":"true",
[junit4] 2> "shards":{
[junit4] 2> "shard2":{
[junit4] 2> "replicas":{
[junit4] 2> "core_node6":{
[junit4] 2> "core":"testNodeWithMultipleReplicasLost_shard2_replica_n6",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10002_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0},
[junit4] 2> "core_node4":{
[junit4] 2> "core":"testNodeWithMultipleReplicasLost_shard2_replica_n4",
[junit4] 2> "leader":"true",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10000_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0},
[junit4] 2> "core_node5":{
[junit4] 2> "core":"testNodeWithMultipleReplicasLost_shard2_replica_n5",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10001_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0}},
[junit4] 2> "range":"0-7fffffff",
[junit4] 2> "state":"active"},
[junit4] 2> "shard1":{
[junit4] 2> "replicas":{
[junit4] 2> "core_node1":{
[junit4] 2> "core":"testNodeWithMultipleReplicasLost_shard1_replica_n1",
[junit4] 2> "leader":"true",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10003_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0},
[junit4] 2> "core_node2":{
[junit4] 2> "core":"testNodeWithMultipleReplicasLost_shard1_replica_n2",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10002_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0},
[junit4] 2> "core_node3":{
[junit4] 2> "core":"testNodeWithMultipleReplicasLost_shard1_replica_n3",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10001_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0}},
[junit4] 2> "range":"80000000-ffffffff",
[junit4] 2> "state":"active"}}}}
[junit4] 2> 180483 DEBUG (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.s.SimClusterStateProvider -- already has leader testNodeWithMultipleReplicasLost / shard2
[junit4] 2> 180483 DEBUG (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.s.SimClusterStateProvider -- already has leader testNodeWithMultipleReplicasLost / shard1
[junit4] 2> 180486 DEBUG (ScheduledTrigger-885-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
[junit4] 2> 180486 DEBUG (ScheduledTrigger-885-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10001_solr
[junit4] 2> 180496 DEBUG (ScheduledTrigger-885-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 3
[junit4] 2> 180497 DEBUG (ScheduledTrigger-885-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10001_solr
[junit4] 2> 180506 DEBUG (ScheduledTrigger-885-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
[junit4] 2> 180517 DEBUG (ScheduledTrigger-885-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 3
[junit4] 2> 180517 DEBUG (ScheduledTrigger-885-thread-4) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:10001_solr]
[junit4] 2> 180519 DEBUG (ScheduledTrigger-885-thread-4) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
[junit4] 2> "id":"e84e64ced2b9aT5wmnteurs7z6bm7dh932g1pjb",
[junit4] 2> "source":"node_lost_trigger",
[junit4] 2> "eventTime":4086774341905306,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[4086774341905306],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "nodeNames":["127.0.0.1:10001_solr"]}}
[junit4] 2> 180520 DEBUG (ScheduledTrigger-885-thread-4) [ ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, .scheduled_maintenance, node_lost_trigger]
[junit4] 2> 180535 DEBUG (simCloudManagerPool-884-thread-13) [ ] o.a.s.c.o.ClusterStateMutator building a new cName: .system
[junit4] 2> 180548 INFO (simCloudManagerPool-884-thread-13) [ ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 4086776913 sessionWrapper.createTime 4086776881946406, this.sessionWrapper.createTime 4086776881946406
[junit4] 2> 180560 DEBUG (simCloudManagerPool-884-thread-14) [ ] o.a.s.c.a.s.SimClusterStateProvider -- submit leader election for .system / shard1
[junit4] 2> 180573 DEBUG (simCloudManagerPool-884-thread-15) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election for .system / shard1
[junit4] 2> 180573 DEBUG (simCloudManagerPool-884-thread-15) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for .system / shard1: core_node1
[junit4] 2> 180587 DEBUG (AutoscalingActionExecutor-886-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
[junit4] 2> "id":"e84e64ced2b9aT5wmnteurs7z6bm7dh932g1pjb",
[junit4] 2> "source":"node_lost_trigger",
[junit4] 2> "eventTime":4086774341905306,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[4086774341905306],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":4086775513871456,
[junit4] 2> "nodeNames":["127.0.0.1:10001_solr"]}}
[junit4] 2> 180588 DEBUG (AutoscalingActionExecutor-886-thread-1) [ ] o.a.s.c.a.ComputePlanAction -- processing event: {
[junit4] 2> "id":"e84e64ced2b9aT5wmnteurs7z6bm7dh932g1pjb",
[junit4] 2> "source":"node_lost_trigger",
[junit4] 2> "eventTime":4086774341905306,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[4086774341905306],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":4086775513871456,
[junit4] 2> "nodeNames":["127.0.0.1:10001_solr"]}} with context properties: {BEFORE_ACTION=[compute_plan]}
[junit4] 2> 180594 INFO (AutoscalingActionExecutor-886-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testNodeWithMultipleReplicasLost&targetNode=127.0.0.1:10003_solr&inPlaceMove=true&replica=core_node5
[junit4] 2> 180595 INFO (AutoscalingActionExecutor-886-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testNodeWithMultipleReplicasLost&targetNode=127.0.0.1:10000_solr&inPlaceMove=true&replica=core_node3
[junit4] 2> 180595 INFO (AutoscalingActionExecutor-886-thread-1) [ ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 4086779264 sessionWrapper.createTime 4086779013489956, this.sessionWrapper.createTime 4086779013489956
[junit4] 2> 180597 INFO (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.s.TestSimComputePlanAction action=MOVEREPLICA&collection=testNodeWithMultipleReplicasLost&targetNode=127.0.0.1:10003_solr&inPlaceMove=true&replica=core_node5
[junit4] 2> 180597 INFO (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.s.TestSimComputePlanAction action=MOVEREPLICA&collection=testNodeWithMultipleReplicasLost&targetNode=127.0.0.1:10000_solr&inPlaceMove=true&replica=core_node3
[junit4] 2> 180598 INFO (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.s.TestSimComputePlanAction -------------_ FINAL STATE --------------
[junit4] 2> 180598 INFO (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.s.TestSimComputePlanAction * Node values: {
[junit4] 2> "127.0.0.1:10001_solr":{
[junit4] 2> "sysprop.java.vendor":"Oracle Corporation",
[junit4] 2> "cores":2,
[junit4] 2> "host":"127.0.0.1",
[junit4] 2> "freedisk":1022,
[junit4] 2> "sysprop.java.version":"1.8.0_172",
[junit4] 2> "ip_1":"127",
[junit4] 2> "ip_3":"0",
[junit4] 2> "ip_2":"0",
[junit4] 2> "ip_4":"1",
[junit4] 2> "node":"127.0.0.1:10001_solr",
[junit4] 2> "metrics:solr.jvm:buffers.direct.Count":0,
[junit4] 2> "port":10001,
[junit4] 2> "heapUsage":123450000,
[junit4] 2> "sysLoadAvg":1.0,
[junit4] 2> "metrics:solr.node:ADMIN./admin/authorization.clientErrors:count":0},
[junit4] 2> "127.0.0.1:10000_solr":{
[junit4] 2> "sysprop.java.vendor":"Oracle Corporation",
[junit4] 2> "cores":1,
[junit4] 2> "host":"127.0.0.1",
[junit4] 2> "freedisk":999,
[junit4] 2> "sysprop.java.version":"1.8.0_172",
[junit4] 2> "ip_1":"127",
[junit4] 2> "ip_3":"0",
[junit4] 2> "ip_2":"0",
[junit4] 2> "ip_4":"1",
[junit4] 2> "node":"127.0.0.1:10000_solr",
[junit4] 2> "metrics:solr.jvm:buffers.direct.Count":0,
[junit4] 2> "port":10000,
[junit4] 2> "heapUsage":123450000,
[junit4] 2> "sysLoadAvg":1.0,
[junit4] 2> "metrics:solr.node:ADMIN./admin/authorization.clientErrors:count":0},
[junit4] 2> "127.0.0.1:10002_solr":{
[junit4] 2> "sysprop.java.vendor":"Oracle Corporation",
[junit4] 2> "cores":2,
[junit4] 2> "host":"127.0.0.1",
[junit4] 2> "freedisk":1022,
[junit4] 2> "sysprop.java.version":"1.8.0_172",
[junit4] 2> "ip_1":"127",
[junit4] 2> "ip_3":"0",
[junit4] 2> "ip_2":"0",
[junit4] 2> "ip_4":"1",
[junit4] 2> "node":"127.0.0.1:10002_solr",
[junit4] 2> "metrics:solr.jvm:buffers.direct.Count":0,
[junit4] 2> "port":10002,
[junit4] 2> "heapUsage":123450000,
[junit4] 2> "sysLoadAvg":1.0,
[junit4] 2> "metrics:solr.node:ADMIN./admin/authorization.clientErrors:count":0},
[junit4] 2> "127.0.0.1:10003_solr":{
[junit4] 2> "sysprop.java.vendor":"Oracle Corporation",
[junit4] 2> "cores":2,
[junit4] 2> "host":"127.0.0.1",
[junit4] 2> "freedisk":1022,
[junit4] 2> "sysprop.java.version":"1.8.0_172",
[junit4] 2> "ip_1":"127",
[junit4] 2> "ip_3":"0",
[junit4] 2> "ip_2":"0",
[junit4] 2> "ip_4":"1",
[junit4] 2> "node":"127.0.0.1:10003_solr",
[junit4] 2> "metrics:solr.jvm:buffers.direct.Count":0,
[junit4] 2> "port":10003,
[junit4] 2> "heapUsage":123450000,
[junit4] 2> "sysLoadAvg":1.0,
[junit4] 2> "metrics:solr.node:ADMIN./admin/authorization.clientErrors:count":0}}
[junit4] 2> 180598 INFO (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.s.TestSimComputePlanAction * Live nodes: [127.0.0.1:10000_solr, 127.0.0.1:10002_solr, 127.0.0.1:10003_solr]
[junit4] 2> 180598 INFO (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.s.TestSimComputePlanAction * Collection .system state: DocCollection(.system//clusterstate.json/17)={
[junit4] 2> "replicationFactor":"1",
[junit4] 2> "pullReplicas":"0",
[junit4] 2> "router":{"name":"compositeId"},
[junit4] 2> "maxShardsPerNode":"1",
[junit4] 2> "autoAddReplicas":"false",
[junit4] 2> "nrtReplicas":"1",
[junit4] 2> "tlogReplicas":"0",
[junit4] 2> "autoCreated":"true",
[junit4] 2> "shards":{"shard1":{
[junit4] 2> "replicas":{"core_node1":{
[junit4] 2> "core":".system_shard1_replica_n1",
[junit4] 2> "leader":"true",
[junit4] 2> "SEARCHER.searcher.maxDoc":2,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":11240,
[junit4] 2> "node_name":"127.0.0.1:10003_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":1.0468065738677979E-5,
[junit4] 2> "SEARCHER.searcher.numDocs":2}},
[junit4] 2> "range":"80000000-7fffffff",
[junit4] 2> "state":"active"}}}
[junit4] 2> 180599 INFO (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.s.TestSimComputePlanAction * Collection testNodeWithMultipleReplicasLost state: DocCollection(testNodeWithMultipleReplicasLost//clusterstate.json/17)={
[junit4] 2> "replicationFactor":"1",
[junit4] 2> "pullReplicas":"0",
[junit4] 2> "router":{"name":"compositeId"},
[junit4] 2> "maxShardsPerNode":"1",
[junit4] 2> "autoAddReplicas":"false",
[junit4] 2> "nrtReplicas":"3",
[junit4] 2> "tlogReplicas":"0",
[junit4] 2> "autoCreated":"true",
[junit4] 2> "shards":{
[junit4] 2> "shard2":{
[junit4] 2> "replicas":{
[junit4] 2> "core_node6":{
[junit4] 2> "core":"testNodeWithMultipleReplicasLost_shard2_replica_n6",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10002_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0},
[junit4] 2> "core_node4":{
[junit4] 2> "core":"testNodeWithMultipleReplicasLost_shard2_replica_n4",
[junit4] 2> "leader":"true",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10000_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0},
[junit4] 2> "core_node5":{
[junit4] 2> "core":"testNodeWithMultipleReplicasLost_shard2_replica_n5",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10001_solr",
[junit4] 2> "state":"down",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0}},
[junit4] 2> "range":"0-7fffffff",
[junit4] 2> "state":"active"},
[junit4] 2> "shard1":{
[junit4] 2> "replicas":{
[junit4] 2> "core_node1":{
[junit4] 2> "core":"testNodeWithMultipleReplicasLost_shard1_replica_n1",
[junit4] 2> "leader":"true",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10003_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0},
[junit4] 2> "core_node2":{
[junit4] 2> "core":"testNodeWithMultipleReplicasLost_shard1_replica_n2",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10002_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0},
[junit4] 2> "core_node3":{
[junit4] 2> "core":"testNodeWithMultipleReplicasLost_shard1_replica_n3",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10001_solr",
[junit4] 2> "state":"down",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0}},
[junit4] 2> "range":"80000000-ffffffff",
[junit4] 2> "state":"active"}}}
[junit4] 2> 180599 INFO (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[69F3AA28CE054E1]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testNodeWithMultipleReplicasLost
[junit4] 2> 180602 INFO (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase #######################################
[junit4] 2> ############ CLUSTER STATE ############
[junit4] 2> #######################################
[junit4] 2> ## Live nodes: 3
[junit4] 2> ## Empty nodes: 0
[junit4] 2> ## Dead nodes: 1
[junit4] 2> ## 127.0.0.1:10001_solr
[junit4] 2> ## Collections: [.system, testNodeWithMultipleReplicasLost]
[junit4] 2> ## Max replicas per node: 2
[junit4] 2> ## Min replicas per node: 1
[junit4] 2> ## Total replicas: 5
[junit4] 2> ## * .system 1
[junit4] 2> ## - active 1
[junit4] 2> ## * testNodeWithMultipleReplicasLost 4
[junit4] 2> ## - active 4
[junit4] 2> ######### Solr op counts ##########
[junit4] 2> ## - CREATE 1
[junit4] 2> ## - autoscaling 3
[junit4] 2> ## - update 2
[junit4] 2> ######### Autoscaling event counts ###########
[junit4] 2> ## * Trigger: node_lost_trigger
[junit4] 2> ## - BEFORE_ACTION 1
[junit4] 2> ## - STARTED 1
[junit4] 2>
[junit4] 2> 180811 INFO (TEST-TestSimComputePlanAction.testNodeLost-seed#[69F3AA28CE054E1]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testNodeLost
[junit4] 2> 180811 DEBUG (TEST-TestSimComputePlanAction.testNodeLost-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 6
[junit4] 2> 180812 DEBUG (TEST-TestSimComputePlanAction.testNodeLost-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.s.SimNodeStateProvider - removing dead node values: 127.0.0.1:10001_solr
[junit4] 2> 180812 DEBUG (TEST-TestSimComputePlanAction.testNodeLost-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.s.SimClusterStateProvider --- new Overseer leader: 127.0.0.1:10002_solr
[junit4] 2> 180812 DEBUG (TEST-TestSimComputePlanAction.testNodeLost-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.s.SimNodeStateProvider --removing value for 127.0.0.1:10000_solr
[junit4] 2> 180812 DEBUG (TEST-TestSimComputePlanAction.testNodeLost-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.s.SimClusterStateProvider --- new Overseer leader: 127.0.0.1:10003_solr
[junit4] 2> 180812 DEBUG (TEST-TestSimComputePlanAction.testNodeLost-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.s.SimNodeStateProvider --removing value for 127.0.0.1:10002_solr
[junit4] 2> 180812 INFO (TEST-TestSimComputePlanAction.testNodeLost-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
[junit4] 2> 180812 DEBUG (TEST-TestSimComputePlanAction.testNodeLost-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
[junit4] 2> 180812 DEBUG (TEST-TestSimComputePlanAction.testNodeLost-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now
[junit4] 2> 180812 DEBUG (TEST-TestSimComputePlanAction.testNodeLost-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
[junit4] 2> 180813 DEBUG (simCloudManagerPool-884-thread-18) [ ] o.a.s.c.o.ClusterStateMutator building a new cName: .system
[junit4] 2> 180813 WARN (AutoscalingActionExecutor-886-thread-1) [ ] o.a.s.c.a.SystemLogListener Exception sending event to collection .system
[junit4] 2> java.io.IOException: java.lang.InterruptedException
[junit4] 2> at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.request(SimCloudManager.java:632) ~[test/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.SystemLogListener.onEvent(SystemLogListener.java:118) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers$TriggerListeners.fireListeners(ScheduledTriggers.java:792) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers$TriggerListeners.fireListeners(ScheduledTriggers.java:759) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:321) ~[java/:?]
[junit4] 2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_172]
[junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_172]
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) ~[java/:?]
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_172]
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_172]
[junit4] 2> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_172]
[junit4] 2> Caused by: java.lang.InterruptedException
[junit4] 2> at java.util.concurrent.FutureTask.awaitDone(FutureTask.java:404) ~[?:1.8.0_172]
[junit4] 2> at java.util.concurrent.FutureTask.get(FutureTask.java:191) ~[?:1.8.0_172]
[junit4] 2> at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.request(SimCloudManager.java:630) ~[test/:?]
[junit4] 2> ... 10 more
[junit4] 2> 180815 DEBUG (AutoscalingActionExecutor-886-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 11419 ms for event id=e84e64ced2b9aT5wmnteurs7z6bm7dh932g1pjb
[junit4] 2> 180815 DEBUG (TEST-TestSimComputePlanAction.testNodeLost-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor
[junit4] 2> 180815 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 180816 DEBUG (TEST-TestSimComputePlanAction.testNodeLost-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
[junit4] 2> 180816 DEBUG (TEST-TestSimComputePlanAction.testNodeLost-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
[junit4] 2> 180821 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
[junit4] 2> 180821 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 7
[junit4] 2> 180822 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 7, lastZnodeVersion -1
[junit4] 2> 180822 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 7
[junit4] 2> 180833 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10003_solr]
[junit4] 2> 180833 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 180833 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 7, lastZnodeVersion 7
[junit4] 2> 180833 DEBUG (ScheduledTrigger-888-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 180862 DEBUG (ScheduledTrigger-888-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 180882 DEBUG (ScheduledTrigger-888-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 180903 DEBUG (ScheduledTrigger-888-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 180925 DEBUG (ScheduledTrigger-888-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 180929 DEBUG (simCloudManagerPool-887-thread-1) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 180929 DEBUG (simCloudManagerPool-887-thread-1) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 8
[junit4] 2> 180930 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 8
[junit4] 2> 180930 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 180930 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 8, lastZnodeVersion 8
[junit4] 2> 180942 DEBUG (simCloudManagerPool-887-thread-2) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 180942 DEBUG (simCloudManagerPool-887-thread-2) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 9
[junit4] 2> 180943 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 9
[junit4] 2> 180943 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 180943 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 9, lastZnodeVersion 9
[junit4] 2> 180947 DEBUG (ScheduledTrigger-888-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 180968 DEBUG (ScheduledTrigger-888-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 180988 DEBUG (ScheduledTrigger-888-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 181012 DEBUG (ScheduledTrigger-888-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 181032 DEBUG (ScheduledTrigger-888-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 181056 DEBUG (ScheduledTrigger-888-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 181056 DEBUG (simCloudManagerPool-887-thread-3) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 181057 DEBUG (simCloudManagerPool-887-thread-3) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 10
[junit4] 2> 181057 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 10
[junit4] 2> 181058 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger node_lost_trigger - Initial livenodes: [127.0.0.1:10004_solr, 127.0.0.1:10003_solr]
[junit4] 2> 181058 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 181058 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 10, lastZnodeVersion 10
[junit4] 2> 181058 DEBUG (ScheduledTrigger-888-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2
[junit4] 2> 181058 DEBUG (simCloudManagerPool-887-thread-4) [ ] o.a.s.c.o.ClusterStateMutator building a new cName: testNodeLost
[junit4] 2> 181070 INFO (simCloudManagerPool-887-thread-4) [ ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 4086803012 sessionWrapper.createTime 4086802966565156, this.sessionWrapper.createTime 4086802966565156
[junit4] 2> 181076 DEBUG (ScheduledTrigger-888-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 181078 DEBUG (ScheduledTrigger-888-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2
[junit4] 2> 181081 DEBUG (simCloudManagerPool-887-thread-5) [ ] o.a.s.c.a.s.SimClusterStateProvider -- submit leader election for testNodeLost / shard1
[junit4] 2> 181088 DEBUG (simCloudManagerPool-887-thread-7) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election for testNodeLost / shard1
[junit4] 2> 181088 DEBUG (simCloudManagerPool-887-thread-7) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeLost / shard1: core_node1
[junit4] 2> 181096 DEBUG (ScheduledTrigger-888-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 181098 DEBUG (simCloudManagerPool-887-thread-6) [ ] o.a.s.c.a.s.SimClusterStateProvider -- already has leader testNodeLost / shard1
[junit4] 2> 181098 DEBUG (TEST-TestSimComputePlanAction.testNodeLost-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.s.TestSimComputePlanAction -- cluster state: znodeVersion: 25
[junit4] 2> live nodes:[127.0.0.1:10003_solr, 127.0.0.1:10004_solr]
[junit4] 2> collections:{testNodeLost=DocCollection(testNodeLost//clusterstate.json/24)={
[junit4] 2> "replicationFactor":"1",
[junit4] 2> "pullReplicas":"0",
[junit4] 2> "router":{"name":"compositeId"},
[junit4] 2> "maxShardsPerNode":"1",
[junit4] 2> "autoAddReplicas":"false",
[junit4] 2> "nrtReplicas":"2",
[junit4] 2> "tlogReplicas":"0",
[junit4] 2> "autoCreated":"true",
[junit4] 2> "shards":{"shard1":{
[junit4] 2> "replicas":{
[junit4] 2> "core_node1":{
[junit4] 2> "core":"testNodeLost_shard1_replica_n1",
[junit4] 2> "leader":"true",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10004_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0},
[junit4] 2> "core_node2":{
[junit4] 2> "core":"testNodeLost_shard1_replica_n2",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10003_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0}},
[junit4] 2> "range":"80000000-7fffffff",
[junit4] 2> "state":"active"}}}}
[junit4] 2> 181099 DEBUG (TEST-TestSimComputePlanAction.testNodeLost-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.s.SimClusterStateProvider -- submit leader election for testNodeLost / shard1
[junit4] 2> 181101 INFO (TEST-TestSimComputePlanAction.testNodeLost-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.s.TestSimComputePlanAction Stopped_node : 127.0.0.1:10004_solr
[junit4] 2> 181109 DEBUG (ScheduledTrigger-888-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2
[junit4] 2> 181109 DEBUG (ScheduledTrigger-888-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10004_solr
[junit4] 2> 181112 DEBUG (simCloudManagerPool-887-thread-8) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election for testNodeLost / shard1
[junit4] 2> 181115 INFO (simCloudManagerPool-887-thread-8) [ ] o.a.s.c.ActionThrottle Throttling leader attempts - waiting for 3637ms
[junit4] 2> 181117 DEBUG (ScheduledTrigger-888-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 181117 DEBUG (ScheduledTrigger-888-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10004_solr
[junit4] 2> 181130 DEBUG (ScheduledTrigger-888-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2
[junit4] 2> 181137 DEBUG (ScheduledTrigger-888-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 181150 DEBUG (ScheduledTrigger-888-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2
[junit4] 2> 181157 DEBUG (ScheduledTrigger-888-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 181170 DEBUG (ScheduledTrigger-888-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2
[junit4] 2> 181177 DEBUG (ScheduledTrigger-888-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 181189 DEBUG (simCloudManagerPool-887-thread-8) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeLost / shard1: core_node2
[junit4] 2> 181190 DEBUG (ScheduledTrigger-888-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2
[junit4] 2> 181198 DEBUG (ScheduledTrigger-888-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 181210 DEBUG (ScheduledTrigger-888-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2
[junit4] 2> 181219 DEBUG (ScheduledTrigger-888-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 181231 DEBUG (ScheduledTrigger-888-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2
[junit4] 2> 181239 DEBUG (ScheduledTrigger-888-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 181251 DEBUG (ScheduledTrigger-888-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2
[junit4] 2> 181251 DEBUG (ScheduledTrigger-888-thread-1) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:10004_solr]
[junit4] 2> 181251 DEBUG (ScheduledTrigger-888-thread-1) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
[junit4] 2> "id":"e84ed6f3e6a90T5wmnteurs7z6bm7dh932g1pjg",
[junit4] 2> "source":"node_lost_trigger",
[junit4] 2> "eventTime":4086804982426256,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[4086804982426256],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "nodeNames":["127.0.0.1:10004_solr"]}}
[junit4] 2> 181251 DEBUG (ScheduledTrigger-888-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, .scheduled_maintenance, node_lost_trigger]
[junit4] 2> 181261 DEBUG (simCloudManagerPool-887-thread-9) [ ] o.a.s.c.o.ClusterStateMutator building a new cName: .system
[junit4] 2> 181273 INFO (simCloudManagerPool-887-thread-9) [ ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 4086813147 sessionWrapper.createTime 4086813122641706, this.sessionWrapper.createTime 4086813122641706
[junit4] 2> 181295 DEBUG (simCloudManagerPool-887-thread-10) [ ] o.a.s.c.a.s.SimClusterStateProvider -- submit leader election for .system / shard1
[junit4] 2> 181296 DEBUG (simCloudManagerPool-887-thread-11) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election for .system / shard1
[junit4] 2> 181297 DEBUG (simCloudManagerPool-887-thread-11) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for .system / shard1: core_node2
[junit4] 2> 181607 DEBUG (AutoscalingActionExecutor-889-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
[junit4] 2> "id":"e84ed6f3e6a90T5wmnteurs7z6bm7dh932g1pjg",
[junit4] 2> "source":"node_lost_trigger",
[junit4] 2> "eventTime":4086804982426256,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[4086804982426256],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":4086812087975156,
[junit4] 2> "nodeNames":["127.0.0.1:10004_solr"]}}
[junit4] 2> 181607 DEBUG (AutoscalingActionExecutor-889-thread-1) [ ] o.a.s.c.a.ComputePlanAction -- processing event: {
[junit4] 2> "id":"e84ed6f3e6a90T5wmnteurs7z6bm7dh932g1pjg",
[junit4] 2> "source":"node_lost_trigger",
[junit4] 2> "eventTime":4086804982426256,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[4086804982426256],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":4086812087975156,
[junit4] 2> "nodeNames":["127.0.0.1:10004_solr"]}} with context properties: {BEFORE_ACTION=[compute_plan]}
[junit4] 2> 181611 INFO (AutoscalingActionExecutor-889-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testNodeLost&targetNode=127.0.0.1:10005_solr&inPlaceMove=true&replica=core_node1
[junit4] 2> 181613 INFO (AutoscalingActionExecutor-889-thread-1) [ ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 4086830142 sessionWrapper.createTime 4086829940143706, this.sessionWrapper.createTime 4086829940143706
[junit4] 2> 181615 DEBUG (TEST-TestSimComputePlanAction.testNodeLost-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.s.SimClusterStateProvider -- submit leader election for .system / shard1
[junit4] 2> 181623 INFO (TEST-TestSimComputePlanAction.testNodeLost-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.s.TestSimComputePlanAction -------------_ FINAL STATE --------------
[junit4] 2> 181623 DEBUG (simCloudManagerPool-887-thread-13) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election for .system / shard1
[junit4] 2> 181623 INFO (TEST-TestSimComputePlanAction.testNodeLost-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.s.TestSimComputePlanAction * Node values: {
[junit4] 2> "127.0.0.1:10004_solr":{
[junit4] 2> "sysprop.java.vendor":"Oracle Corporation",
[junit4] 2> "cores":1,
[junit4] 2> "host":"127.0.0.1",
[junit4] 2> "freedisk":1023,
[junit4] 2> "sysprop.java.version":"1.8.0_172",
[junit4] 2> "ip_1":"127",
[junit4] 2> "ip_3":"0",
[junit4] 2> "ip_2":"0",
[junit4] 2> "ip_4":"1",
[junit4] 2> "node":"127.0.0.1:10004_solr",
[junit4] 2> "metrics:solr.jvm:buffers.direct.Count":0,
[junit4] 2> "port":10004,
[junit4] 2> "heapUsage":123450000,
[junit4] 2> "sysLoadAvg":1.0,
[junit4] 2> "metrics:solr.node:ADMIN./admin/authorization.clientErrors:count":0},
[junit4] 2> "127.0.0.1:10005_solr":{
[junit4] 2> "sysprop.java.vendor":"Oracle Corporation",
[junit4] 2> "cores":1,
[junit4] 2> "host":"127.0.0.1",
[junit4] 2> "freedisk":1023,
[junit4] 2> "sysprop.java.version":"1.8.0_172",
[junit4] 2> "ip_1":"127",
[junit4] 2> "ip_3":"0",
[junit4] 2> "ip_2":"0",
[junit4] 2> "ip_4":"1",
[junit4] 2> "node":"127.0.0.1:10005_solr",
[junit4] 2> "metrics:solr.jvm:buffers.direct.Count":0,
[junit4] 2> "port":10005,
[junit4] 2> "heapUsage":123450000,
[junit4] 2> "sysLoadAvg":1.0,
[junit4] 2> "metrics:solr.node:ADMIN./admin/authorization.clientErrors:count":0},
[junit4] 2> "127.0.0.1:10003_solr":{
[junit4] 2> "sysprop.java.vendor":"Oracle Corporation",
[junit4] 2> "cores":1,
[junit4] 2> "host":"127.0.0.1",
[junit4] 2> "freedisk":999,
[junit4] 2> "sysprop.java.version":"1.8.0_172",
[junit4] 2> "ip_1":"127",
[junit4] 2> "ip_3":"0",
[junit4] 2> "ip_2":"0",
[junit4] 2> "ip_4":"1",
[junit4] 2> "node":"127.0.0.1:10003_solr",
[junit4] 2> "metrics:solr.jvm:buffers.direct.Count":0,
[junit4] 2> "port":10003,
[junit4] 2> "heapUsage":123450000,
[junit4] 2> "sysLoadAvg":1.0,
[junit4] 2> "metrics:solr.node:ADMIN./admin/authorization.clientErrors:count":0}}
[junit4] 2> 181623 INFO (TEST-TestSimComputePlanAction.testNodeLost-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.s.TestSimComputePlanAction * Live nodes: [127.0.0.1:10003_solr]
[junit4] 2> 181623 INFO (TEST-TestSimComputePlanAction.testNodeLost-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.s.TestSimComputePlanAction * Collection .system state: DocCollection(.system//clusterstate.json/32)={
[junit4] 2> "replicationFactor":"1",
[junit4] 2> "pullReplicas":"0",
[junit4] 2> "router":{"name":"compositeId"},
[junit4] 2> "maxShardsPerNode":"1",
[junit4] 2> "autoAddReplicas":"false",
[junit4] 2> "nrtReplicas":"1",
[junit4] 2> "tlogReplicas":"0",
[junit4] 2> "autoCreated":"true",
[junit4] 2> "shards":{"shard1":{
[junit4] 2> "replicas":{"core_node2":{
[junit4] 2> "core":".system_shard1_replica_n1",
[junit4] 2> "SEARCHER.searcher.maxDoc":2,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":11240,
[junit4] 2> "node_name":"127.0.0.1:10005_solr",
[junit4] 2> "state":"down",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":1.0468065738677979E-5,
[junit4] 2> "SEARCHER.searcher.numDocs":2}},
[junit4] 2> "range":"80000000-7fffffff",
[junit4] 2> "state":"active"}}}
[junit4] 2> 181623 INFO (TEST-TestSimComputePlanAction.testNodeLost-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.s.TestSimComputePlanAction * Collection testNodeLost state: DocCollection(testNodeLost//clusterstate.json/32)={
[junit4] 2> "replicationFactor":"1",
[junit4] 2> "pullReplicas":"0",
[junit4] 2> "router":{"name":"compositeId"},
[junit4] 2> "maxShardsPerNode":"1",
[junit4] 2> "autoAddReplicas":"false",
[junit4] 2> "nrtReplicas":"2",
[junit4] 2> "tlogReplicas":"0",
[junit4] 2> "autoCreated":"true",
[junit4] 2> "shards":{"shard1":{
[junit4] 2> "replicas":{
[junit4] 2> "core_node1":{
[junit4] 2> "core":"testNodeLost_shard1_replica_n1",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10004_solr",
[junit4] 2> "state":"down",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0},
[junit4] 2> "core_node2":{
[junit4] 2> "core":"testNodeLost_shard1_replica_n2",
[junit4] 2> "leader":"true",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10003_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0}},
[junit4] 2> "range":"80000000-7fffffff",
[junit4] 2> "state":"active"}}}
[junit4] 2> 181624 INFO (TEST-TestSimComputePlanAction.testNodeLost-seed#[69F3AA28CE054E1]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testNodeLost
[junit4] 2> 181624 INFO (TEST-TestSimComputePlanAction.testNodeLost-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase #######################################
[junit4] 2> ############ CLUSTER STATE ############
[junit4] 2> #######################################
[junit4] 2> ## Live nodes: 1
[junit4] 2> ## Empty nodes: 0
[junit4] 2> ## Dead nodes: 2
[junit4] 2> ## 127.0.0.1:10004_solr
[junit4] 2> ## 127.0.0.1:10005_solr
[junit4] 2> ## Collections: [.system, testNodeLost]
[junit4] 2> ## Max replicas per node: 1
[junit4] 2> ## Min replicas per node: 1
[junit4] 2> ## Total replicas: 1
[junit4] 2> ## * testNodeLost 1
[junit4] 2> ## - active 1
[junit4] 2> ######### Solr op counts ##########
[junit4] 2> ## - CREATE 1
[junit4] 2> ## - autoscaling 3
[junit4] 2> ## - update 2
[junit4] 2> ######### Autoscaling event counts ###########
[junit4] 2> ## * Trigger: node_lost_trigger
[junit4] 2> ## - BEFORE_ACTION 1
[junit4] 2> ## - STARTED 1
[junit4] 2>
[junit4] 2> 181630 INFO (TEST-TestSimComputePlanAction.testNodeAdded-seed#[69F3AA28CE054E1]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testNodeAdded
[junit4] 2> 181630 DEBUG (TEST-TestSimComputePlanAction.testNodeAdded-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 11
[junit4] 2> 181630 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 11
[junit4] 2> 181630 DEBUG (TEST-TestSimComputePlanAction.testNodeAdded-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.s.SimNodeStateProvider - removing dead node values: 127.0.0.1:10004_solr
[junit4] 2> 181630 DEBUG (TEST-TestSimComputePlanAction.testNodeAdded-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.s.SimNodeStateProvider - removing dead node values: 127.0.0.1:10005_solr
[junit4] 2> 181631 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
[junit4] 2> 181631 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 181631 INFO (TEST-TestSimComputePlanAction.testNodeAdded-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
[junit4] 2> 181631 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 11, lastZnodeVersion 11
[junit4] 2> 181631 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Interrupted
[junit4] 2> java.lang.InterruptedException: null
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2034) ~[?:1.8.0_172]
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:189) [java/:?]
[junit4] 2> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_172]
[junit4] 2> 181632 DEBUG (TEST-TestSimComputePlanAction.testNodeAdded-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
[junit4] 2> 181632 DEBUG (TEST-TestSimComputePlanAction.testNodeAdded-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now
[junit4] 2> 181632 DEBUG (TEST-TestSimComputePlanAction.testNodeAdded-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
[junit4] 2> 181632 WARN (AutoscalingActionExecutor-889-thread-1) [ ] o.a.s.c.a.SystemLogListener Exception sending event to collection .system
[junit4] 2> java.io.IOException: java.lang.InterruptedException
[junit4] 2> at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.request(SimCloudManager.java:632) ~[test/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.SystemLogListener.onEvent(SystemLogListener.java:118) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers$TriggerListeners.fireListeners(ScheduledTriggers.java:792) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers$TriggerListeners.fireListeners(ScheduledTriggers.java:768) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:315) ~[java/:?]
[junit4] 2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_172]
[junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_172]
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) ~[java/:?]
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_172]
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_172]
[junit4] 2> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_172]
[junit4] 2> Caused by: java.lang.InterruptedException
[junit4] 2> at java.util.concurrent.FutureTask.awaitDone(FutureTask.java:404) ~[?:1.8.0_172]
[junit4] 2> at java.util.concurrent.FutureTask.get(FutureTask.java:191) ~[?:1.8.0_172]
[junit4] 2> at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.request(SimCloudManager.java:630) ~[test/:?]
[junit4] 2> ... 10 more
[junit4] 2> 181636 DEBUG (simCloudManagerPool-887-thread-15) [ ] o.a.s.c.o.ClusterStateMutator building a new cName: .system
[junit4] 2> 181645 DEBUG (simCloudManagerPool-887-thread-14) [ ] o.a.s.c.o.ClusterStateMutator building a new cName: .system
[junit4] 2> 181625 WARN (simCloudManagerPool-887-thread-13) [ ] o.a.s.c.a.s.SimClusterStateProvider Can't find any active replicas for .system / shard1: shard1:{
[junit4] 2> "replicas":{"core_node2":{
[junit4] 2> "core":".system_shard1_replica_n1",
[junit4] 2> "SEARCHER.searcher.maxDoc":2,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":11240,
[junit4] 2> "node_name":"127.0.0.1:10005_solr",
[junit4] 2> "state":"down",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":1.0468065738677979E-5,
[junit4] 2> "SEARCHER.searcher.numDocs":2}},
[junit4] 2> "range":"80000000-7fffffff",
[junit4] 2> "state":"active"}
[junit4] 2> 181648 DEBUG (simCloudManagerPool-887-thread-13) [ ] o.a.s.c.a.s.SimClusterStateProvider -- liveNodes: [127.0.0.1:10003_solr]
[junit4] 2> 181657 DEBUG (simCloudManagerPool-887-thread-16) [ ] o.a.s.c.a.s.SimClusterStateProvider -- submit leader election for .system / shard1
[junit4] 2> 181664 DEBUG (simCloudManagerPool-887-thread-17) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election for .system / shard1
[junit4] 2> 181664 DEBUG (simCloudManagerPool-887-thread-17) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for .system / shard1: core_node3
[junit4] 2> 181665 WARN (AutoscalingActionExecutor-889-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Unhandled exception executing actions
[junit4] 2> org.apache.lucene.store.AlreadyClosedException: ScheduledTrigger node_lost_trigger has been closed.
[junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers$TriggerWrapper.dequeue(ScheduledTriggers.java:548) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:318) ~[java/:?]
[junit4] 2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_172]
[junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_172]
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) ~[java/:?]
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_172]
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_172]
[junit4] 2> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_172]
[junit4] 2> 181666 DEBUG (AutoscalingActionExecutor-889-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 2928 ms for event id=e84ed6f3e6a90T5wmnteurs7z6bm7dh932g1pjg
[junit4] 2> 181666 DEBUG (TEST-TestSimComputePlanAction.testNodeAdded-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor
[junit4] 2> 181666 DEBUG (TEST-TestSimComputePlanAction.testNodeAdded-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
[junit4] 2> 181666 DEBUG (TEST-TestSimComputePlanAction.testNodeAdded-seed#[69F3AA28CE054E1]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
[junit4] 2> 181674 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
[junit4] 2> 181675 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 12
[junit4] 2> 181676 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 12, lastZnodeVersion -1
[junit4] 2> 181676 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 12
[junit4] 2> 181677 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10003_solr]
[junit4] 2> 181678 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 181678 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 12, lastZnodeVersion 12
[junit4] 2> 181678 DEBUG (ScheduledTrigger-891-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 181699 DEBUG (ScheduledTrigger-891-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 181719 DEBUG (ScheduledTrigger-891-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 181739 DEBUG (ScheduledTrigger-891-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 181760 DEBUG (ScheduledTrigger-891-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 181773 DEBUG (simCloudManagerPool-890-thread-1) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 181774 DEBUG (simCloudManagerPool-890-thread-1) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 13
[junit4] 2> 181774 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 13
[junit4] 2> 181775 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 181775 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 13, lastZnodeVersion 13
[junit4] 2> 181777 DEBUG (simCloudManagerPool-890-thread-2) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 181777 DEBUG (simCloudManagerPool-890-thread-2) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 14
[junit4] 2> 181777 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 14
[junit4] 2> 181777 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 181777 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 14, lastZnodeVersion 14
[junit4] 2> 181783 DEBUG (ScheduledTrigger-891-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 181803 DEBUG (ScheduledTrigger-891-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 181824 DEBUG (ScheduledTrigger-891-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 181844 DEBUG (ScheduledTrigger-891-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 181864 DEBUG (ScheduledTrigger-891-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 181884 DEBUG (ScheduledTrigger-891-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 181902 DEBUG (simCloudManagerPool-890-thread-3) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 181902 DEBUG (simCloudManagerPool-890-thread-3) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 15
[junit4] 2> 181904 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 15
[junit4] 2> 181904 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger - Initial livenodes: [127.0.0.1:10003_solr]
[junit4] 2> 181904 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=compute_plan, class=solr.ComputePlanAction}, {name=test, class=org.apache.solr.cloud.autoscaling.sim.TestSimComputePlanAction$AssertingTriggerAction}], enabled=true}
[junit4] 2> 181904 DEBUG (ScheduledTrigger-891-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 181905 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 15, lastZnodeVersion 15
[junit4] 2> 181905 DEBUG (ScheduledTrigger-891-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 181905 DEBUG (ScheduledTrigger-891-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 1
[junit4] 2> 181906 DEBUG (simCloudManagerPool-890-thread-4) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 181906 DEBUG (simCloudManagerPool-890-thread-4) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 16
[junit4] 2> 181907 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 16
[junit4] 2> 181907 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 16, lastZnodeVersion 16
[junit4] 2> 181923 DEBUG (simCloudManagerPool-890-thread-5) [ ] o.a.s.c.o.ClusterStateMutator building a new cName: testNodeAdded
[junit4] 2> 181925 DEBUG (ScheduledTrigger-891-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 181925 DEBUG (ScheduledTrigger-891-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 181925 DEBUG (ScheduledTrigger-891-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 1
[junit4] 2> 181933 WARN (simCloudManagerPool-890-thread-5) [ ] o.a.s.c.a.c.CreateCollectionCmd Specified number of replicas of 4 on collection testNodeAdded is higher than the number of Solr instances currently live or live and part of your createNodeSet(1). It's unusual to run two replica of the same slice on the same Solr-instance.
[junit4] 2> 181935 INFO (simCloudManagerPool-890-thread-5) [ ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 4086846255 sessionWrapper.createTime 4086846218041506, this.sessionWrapper.createTime 4086846218041506
[junit4] 2> 181945 DEBUG (ScheduledTrigger-891-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 181946 DEBUG (ScheduledTrigger-891-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 181946 DEBUG (ScheduledTrigger-891-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 1
[junit4] 2> 181947 DEBUG (simCloudManagerPool-890-thread-6) [ ] o.a.s.c.a.s.SimClusterStateProvider -- submit leader election for testNodeAdded / shard1
[junit4] 2> 181948 DEBUG (simCloudManagerPool-890-thread-10) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election for testNodeAdded / shard1
[junit4] 2> 181950 DEBUG (simCloudManagerPool-890-thread-10) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeAdded / shard1: core_node1
[junit4] 2> 181958 DEBUG (simCloudManagerPool-890-thread-7) [ ] o.a.s.c.a.s.SimClusterStateProvider -- already has leader testNodeAdded / shard1
[junit4] 2> 181965 DEBUG (ScheduledTrigger-891-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 181966 DEBUG (ScheduledTrigger-891-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 181966 DEBUG (ScheduledTrigger-891-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 1
[junit4] 2> 181969 DEBUG (simCloudManagerPool-890-thread-8) [ ] o.a.s.c.a.s.SimClusterStateProvider -- already has leader testNodeAdded / shard1
[junit4] 2> 181980 DEBUG (simCloudManagerPool-890-thread-9) [ ] o.a.s.c.a.s.SimClusterStateProvider -- already has leader testNodeAdded / shard1
[junit4] 2> 181982 DEBUG (simCloudManagerPool-890-thread-11) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configu
[...truncated too long message...]
ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
jar-checksums:
[mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/null1857984141
[copy] Copying 238 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/null1857984141
[delete] Deleting directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/null1857984141
check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] confs: [default]
[ivy:cachepath] found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 21ms :: artifacts dl 1ms
---------------------------------------------------------------------
| | modules || artifacts |
| conf | number| search|dwnlded|evicted|| number|dwnlded|
---------------------------------------------------------------------
| default | 8 | 0 | 0 | 0 || 8 | 0 |
---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...
-jenkins-base:
BUILD SUCCESSFUL
Total time: 95 minutes 33 seconds
Archiving artifacts
java.lang.InterruptedException: no matches found within 10000
at hudson.FilePath$34.hasMatch(FilePath.java:2678)
at hudson.FilePath$34.invoke(FilePath.java:2557)
at hudson.FilePath$34.invoke(FilePath.java:2547)
at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2918)
Also: hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
at hudson.remoting.Channel.call(Channel.java:955)
at hudson.FilePath.act(FilePath.java:1036)
at hudson.FilePath.act(FilePath.java:1025)
at hudson.FilePath.validateAntFileMask(FilePath.java:2547)
at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
at hudson.model.Build$BuildExecution.post2(Build.java:186)
at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
at hudson.model.Run.execute(Run.java:1819)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
Caused: hudson.FilePath$TunneledInterruptedException
at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2920)
at hudson.remoting.UserRequest.perform(UserRequest.java:212)
at hudson.remoting.UserRequest.perform(UserRequest.java:54)
at hudson.remoting.Request$2.run(Request.java:369)
at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
Caused: java.lang.InterruptedException: java.lang.InterruptedException: no matches found within 10000
at hudson.FilePath.act(FilePath.java:1038)
at hudson.FilePath.act(FilePath.java:1025)
at hudson.FilePath.validateAntFileMask(FilePath.java:2547)
at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
at hudson.model.Build$BuildExecution.post2(Build.java:186)
at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
at hudson.model.Run.execute(Run.java:1819)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)
[JENKINS] Lucene-Solr-Tests-master - Build # 2808 - Still Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-master/2808/
2 tests failed.
FAILED: org.apache.solr.cloud.TestPullReplicaErrorHandling.testCantConnectToLeader
Error Message:
Error from server at http://127.0.0.1:37515/solr: Could not find collection : pull_replica_error_handling_test_cant_connect_to_leader
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:37515/solr: Could not find collection : pull_replica_error_handling_test_cant_connect_to_leader
at __randomizedtesting.SeedInfo.seed([66FC441568EA3EF3:E8F024651DF2BAD9]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1107)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:884)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:817)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
at org.apache.solr.cloud.TestPullReplicaErrorHandling.tearDown(TestPullReplicaErrorHandling.java:128)
at sun.reflect.GeneratedMethodAccessor9.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:993)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:944)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:830)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:880)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:891)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.autoscaling.sim.TestSimComputePlanAction.testNodeAdded
Error Message:
ComputePlanAction should have computed exactly 1 operation, but was: [org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@3d5265b9, org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@3edee03f] expected:<1> but was:<2>
Stack Trace:
java.lang.AssertionError: ComputePlanAction should have computed exactly 1 operation, but was: [org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@3d5265b9, org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@3edee03f] expected:<1> but was:<2>
at __randomizedtesting.SeedInfo.seed([66FC441568EA3EF3:33F1262CA4996F0]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:472)
at org.apache.solr.cloud.autoscaling.sim.TestSimComputePlanAction.testNodeAdded(TestSimComputePlanAction.java:314)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:935)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:971)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:985)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:944)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:830)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:880)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:891)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 13507 lines...]
[junit4] Suite: org.apache.solr.cloud.autoscaling.sim.TestSimComputePlanAction
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.sim.TestSimComputePlanAction_66FC441568EA3EF3-001/init-core-data-001
[junit4] 2> 1183101 WARN (SUITE-TestSimComputePlanAction-seed#[66FC441568EA3EF3]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=9 numCloses=9
[junit4] 2> 1183102 INFO (SUITE-TestSimComputePlanAction-seed#[66FC441568EA3EF3]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 1183103 INFO (SUITE-TestSimComputePlanAction-seed#[66FC441568EA3EF3]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 1183103 INFO (SUITE-TestSimComputePlanAction-seed#[66FC441568EA3EF3]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 1183129 DEBUG (SUITE-TestSimComputePlanAction-seed#[66FC441568EA3EF3]-worker) [ ] o.a.s.c.a.s.SimClusterStateProvider --- new Overseer leader: 127.0.0.1:10000_solr
[junit4] 2> 1183129 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
[junit4] 2> 1183129 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 0
[junit4] 2> 1183130 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion -1
[junit4] 2> 1183130 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 0
[junit4] 2> 1183133 INFO (SUITE-TestSimComputePlanAction-seed#[66FC441568EA3EF3]-worker) [ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 1183141 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10000_solr]
[junit4] 2> 1183142 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 1183142 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion 0
[junit4] 2> 1183143 DEBUG (ScheduledTrigger-4640-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 1183143 INFO (TEST-TestSimComputePlanAction.testNodeLost-seed#[66FC441568EA3EF3]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testNodeLost
[junit4] 2> 1183143 DEBUG (TEST-TestSimComputePlanAction.testNodeLost-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
[junit4] 2> 1183146 INFO (TEST-TestSimComputePlanAction.testNodeLost-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
[junit4] 2> 1183146 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 1183147 DEBUG (TEST-TestSimComputePlanAction.testNodeLost-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
[junit4] 2> 1183147 DEBUG (TEST-TestSimComputePlanAction.testNodeLost-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now
[junit4] 2> 1183148 DEBUG (TEST-TestSimComputePlanAction.testNodeLost-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
[junit4] 2> 1183148 DEBUG (TEST-TestSimComputePlanAction.testNodeLost-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor
[junit4] 2> 1183148 DEBUG (TEST-TestSimComputePlanAction.testNodeLost-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
[junit4] 2> 1183148 DEBUG (TEST-TestSimComputePlanAction.testNodeLost-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
[junit4] 2> 1183148 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
[junit4] 2> 1183149 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2
[junit4] 2> 1183149 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion -1
[junit4] 2> 1183149 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
[junit4] 2> 1183149 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10000_solr]
[junit4] 2> 1183150 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 1183150 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
[junit4] 2> 1183150 DEBUG (ScheduledTrigger-4645-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 1183170 DEBUG (ScheduledTrigger-4645-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 1183191 DEBUG (ScheduledTrigger-4645-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 1183211 DEBUG (ScheduledTrigger-4645-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 1183231 DEBUG (ScheduledTrigger-4645-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 1183252 DEBUG (ScheduledTrigger-4645-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 1183265 DEBUG (simCloudManagerPool-4644-thread-1) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 1183265 DEBUG (simCloudManagerPool-4644-thread-1) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 3
[junit4] 2> 1183266 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
[junit4] 2> 1183266 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 1183266 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
[junit4] 2> 1183267 DEBUG (simCloudManagerPool-4644-thread-2) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 1183267 DEBUG (simCloudManagerPool-4644-thread-2) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 4
[junit4] 2> 1183267 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 4
[junit4] 2> 1183267 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 1183268 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion 4
[junit4] 2> 1183272 DEBUG (ScheduledTrigger-4645-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 1183292 DEBUG (ScheduledTrigger-4645-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 1183312 DEBUG (ScheduledTrigger-4645-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 1183332 DEBUG (ScheduledTrigger-4645-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 1183353 DEBUG (ScheduledTrigger-4645-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 1183371 DEBUG (simCloudManagerPool-4644-thread-3) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 1183372 DEBUG (simCloudManagerPool-4644-thread-3) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 5
[junit4] 2> 1183372 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 5
[junit4] 2> 1183372 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger node_lost_trigger - Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10000_solr]
[junit4] 2> 1183372 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 1183373 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 5, lastZnodeVersion 5
[junit4] 2> 1183373 DEBUG (ScheduledTrigger-4645-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2
[junit4] 2> 1183373 DEBUG (ScheduledTrigger-4645-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 1183374 DEBUG (simCloudManagerPool-4644-thread-4) [ ] o.a.s.c.o.ClusterStateMutator building a new cName: testNodeLost
[junit4] 2> 1183389 DEBUG (simCloudManagerPool-4644-thread-4) [ ] o.a.s.c.s.c.a.PolicyHelper Creating a new session
[junit4] 2> 1183391 DEBUG (simCloudManagerPool-4644-thread-4) [ ] o.a.s.c.s.c.a.PolicyHelper New session created
[junit4] 2> 1183394 DEBUG (ScheduledTrigger-4645-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2
[junit4] 2> 1183394 DEBUG (ScheduledTrigger-4645-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 1183398 INFO (simCloudManagerPool-4644-thread-4) [ ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 4140481577 sessionWrapper.createTime 4140481213360878, this.sessionWrapper.createTime 4140481213360878
[junit4] 2> 1183399 DEBUG (simCloudManagerPool-4644-thread-4) [ ] o.a.s.c.s.c.a.PolicyHelper session set to NULL
[junit4] 2> 1183414 DEBUG (ScheduledTrigger-4645-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2
[junit4] 2> 1183414 DEBUG (ScheduledTrigger-4645-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 1183417 DEBUG (simCloudManagerPool-4644-thread-5) [ ] o.a.s.c.a.s.SimClusterStateProvider -- submit leader election for testNodeLost / shard1
[junit4] 2> 1183418 DEBUG (simCloudManagerPool-4644-thread-7) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election for testNodeLost / shard1
[junit4] 2> 1183419 DEBUG (simCloudManagerPool-4644-thread-7) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeLost / shard1: core_node1
[junit4] 2> 1183429 DEBUG (simCloudManagerPool-4644-thread-6) [ ] o.a.s.c.a.s.SimClusterStateProvider -- already has leader testNodeLost / shard1
[junit4] 2> 1183430 DEBUG (TEST-TestSimComputePlanAction.testNodeLost-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.s.TestSimComputePlanAction -- cluster state: znodeVersion: 6
[junit4] 2> live nodes:[127.0.0.1:10001_solr, 127.0.0.1:10000_solr]
[junit4] 2> collections:{testNodeLost=DocCollection(testNodeLost//clusterstate.json/5)={
[junit4] 2> "replicationFactor":"1",
[junit4] 2> "pullReplicas":"0",
[junit4] 2> "router":{"name":"compositeId"},
[junit4] 2> "maxShardsPerNode":"1",
[junit4] 2> "autoAddReplicas":"false",
[junit4] 2> "nrtReplicas":"2",
[junit4] 2> "tlogReplicas":"0",
[junit4] 2> "autoCreated":"true",
[junit4] 2> "shards":{"shard1":{
[junit4] 2> "replicas":{
[junit4] 2> "core_node1":{
[junit4] 2> "core":"testNodeLost_shard1_replica_n1",
[junit4] 2> "leader":"true",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10001_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0},
[junit4] 2> "core_node2":{
[junit4] 2> "core":"testNodeLost_shard1_replica_n2",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10000_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0}},
[junit4] 2> "range":"80000000-7fffffff",
[junit4] 2> "state":"active"}}}}
[junit4] 2> 1183431 DEBUG (TEST-TestSimComputePlanAction.testNodeLost-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.s.SimClusterStateProvider -- submit leader election for testNodeLost / shard1
[junit4] 2> 1183431 INFO (TEST-TestSimComputePlanAction.testNodeLost-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.s.TestSimComputePlanAction Stopped_node : 127.0.0.1:10001_solr
[junit4] 2> 1183431 DEBUG (simCloudManagerPool-4644-thread-8) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election for testNodeLost / shard1
[junit4] 2> 1183432 INFO (simCloudManagerPool-4644-thread-8) [ ] o.a.s.c.ActionThrottle Throttling leader attempts - waiting for 4383ms
[junit4] 2> 1183434 DEBUG (ScheduledTrigger-4645-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2
[junit4] 2> 1183434 DEBUG (ScheduledTrigger-4645-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 1183434 DEBUG (ScheduledTrigger-4645-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10001_solr
[junit4] 2> 1183434 DEBUG (ScheduledTrigger-4645-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10001_solr
[junit4] 2> 1183455 DEBUG (ScheduledTrigger-4645-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 1183455 DEBUG (ScheduledTrigger-4645-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2
[junit4] 2> 1183475 DEBUG (ScheduledTrigger-4645-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 1183475 DEBUG (ScheduledTrigger-4645-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2
[junit4] 2> 1183495 DEBUG (ScheduledTrigger-4645-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 1183495 DEBUG (ScheduledTrigger-4645-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2
[junit4] 2> 1183515 DEBUG (ScheduledTrigger-4645-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 1183515 DEBUG (ScheduledTrigger-4645-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2
[junit4] 2> 1183520 DEBUG (simCloudManagerPool-4644-thread-8) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeLost / shard1: core_node2
[junit4] 2> 1183536 DEBUG (ScheduledTrigger-4645-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 1183536 DEBUG (ScheduledTrigger-4645-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2
[junit4] 2> 1183556 DEBUG (ScheduledTrigger-4645-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 1183556 DEBUG (ScheduledTrigger-4645-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2
[junit4] 2> 1183576 DEBUG (ScheduledTrigger-4645-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 1183576 DEBUG (ScheduledTrigger-4645-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2
[junit4] 2> 1183576 DEBUG (ScheduledTrigger-4645-thread-3) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:10001_solr]
[junit4] 2> 1183577 DEBUG (ScheduledTrigger-4645-thread-3) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
[junit4] 2> "id":"eb5bf68a9c73cTw8ua0pe7uz880lqx2hc4v3nh",
[junit4] 2> "source":"node_lost_trigger",
[junit4] 2> "eventTime":4140483373287228,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[4140483373287228],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "nodeNames":["127.0.0.1:10001_solr"]}}
[junit4] 2> 1183577 DEBUG (ScheduledTrigger-4645-thread-3) [ ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, .scheduled_maintenance, node_lost_trigger]
[junit4] 2> 1183581 DEBUG (simCloudManagerPool-4644-thread-9) [ ] o.a.s.c.o.ClusterStateMutator building a new cName: .system
[junit4] 2> 1183592 DEBUG (simCloudManagerPool-4644-thread-9) [ ] o.a.s.c.s.c.a.PolicyHelper Creating a new session
[junit4] 2> 1183594 DEBUG (simCloudManagerPool-4644-thread-9) [ ] o.a.s.c.s.c.a.PolicyHelper New session created
[junit4] 2> 1183595 INFO (simCloudManagerPool-4644-thread-9) [ ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 4140491397 sessionWrapper.createTime 4140491374158128, this.sessionWrapper.createTime 4140491374158128
[junit4] 2> 1183595 DEBUG (simCloudManagerPool-4644-thread-9) [ ] o.a.s.c.s.c.a.PolicyHelper session set to NULL
[junit4] 2> 1183606 DEBUG (simCloudManagerPool-4644-thread-10) [ ] o.a.s.c.a.s.SimClusterStateProvider -- submit leader election for .system / shard1
[junit4] 2> 1183606 DEBUG (simCloudManagerPool-4644-thread-11) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election for .system / shard1
[junit4] 2> 1183607 DEBUG (simCloudManagerPool-4644-thread-11) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for .system / shard1: core_node1
[junit4] 2> 1183611 DEBUG (AutoscalingActionExecutor-4646-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
[junit4] 2> "id":"eb5bf68a9c73cTw8ua0pe7uz880lqx2hc4v3nh",
[junit4] 2> "source":"node_lost_trigger",
[junit4] 2> "eventTime":4140483373287228,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[4140483373287228],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":4140490525478928,
[junit4] 2> "nodeNames":["127.0.0.1:10001_solr"]}}
[junit4] 2> 1183612 DEBUG (AutoscalingActionExecutor-4646-thread-1) [ ] o.a.s.c.a.ComputePlanAction -- processing event: {
[junit4] 2> "id":"eb5bf68a9c73cTw8ua0pe7uz880lqx2hc4v3nh",
[junit4] 2> "source":"node_lost_trigger",
[junit4] 2> "eventTime":4140483373287228,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[4140483373287228],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":4140490525478928,
[junit4] 2> "nodeNames":["127.0.0.1:10001_solr"]}} with context properties: {BEFORE_ACTION=[compute_plan]}
[junit4] 2> 1183612 DEBUG (AutoscalingActionExecutor-4646-thread-1) [ ] o.a.s.c.s.c.a.PolicyHelper Creating a new session
[junit4] 2> 1183617 DEBUG (AutoscalingActionExecutor-4646-thread-1) [ ] o.a.s.c.s.c.a.PolicyHelper New session created
[junit4] 2> 1183619 INFO (AutoscalingActionExecutor-4646-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testNodeLost&targetNode=127.0.0.1:10002_solr&inPlaceMove=true&replica=core_node1
[junit4] 2> 1183619 INFO (AutoscalingActionExecutor-4646-thread-1) [ ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 4140492605 sessionWrapper.createTime 4140492525944228, this.sessionWrapper.createTime 4140492525944228
[junit4] 2> 1183619 DEBUG (AutoscalingActionExecutor-4646-thread-1) [ ] o.a.s.c.s.c.a.PolicyHelper session set to NULL
[junit4] 2> 1183621 DEBUG (TEST-TestSimComputePlanAction.testNodeLost-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.s.SimClusterStateProvider -- submit leader election for .system / shard1
[junit4] 2> 1183626 INFO (TEST-TestSimComputePlanAction.testNodeLost-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.s.TestSimComputePlanAction -------------_ FINAL STATE --------------
[junit4] 2> 1183626 INFO (TEST-TestSimComputePlanAction.testNodeLost-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.s.TestSimComputePlanAction * Node values: {
[junit4] 2> "127.0.0.1:10001_solr":{
[junit4] 2> "sysprop.java.vendor":"Oracle Corporation",
[junit4] 2> "cores":1,
[junit4] 2> "host":"127.0.0.1",
[junit4] 2> "freedisk":1023,
[junit4] 2> "sysprop.java.version":"1.8.0_172",
[junit4] 2> "ip_1":"127",
[junit4] 2> "ip_3":"0",
[junit4] 2> "ip_2":"0",
[junit4] 2> "ip_4":"1",
[junit4] 2> "node":"127.0.0.1:10001_solr",
[junit4] 2> "metrics:solr.jvm:buffers.direct.Count":0,
[junit4] 2> "port":10001,
[junit4] 2> "heapUsage":123450000,
[junit4] 2> "sysLoadAvg":1.0,
[junit4] 2> "metrics:solr.node:ADMIN./admin/authorization.clientErrors:count":0},
[junit4] 2> "127.0.0.1:10000_solr":{
[junit4] 2> "sysprop.java.vendor":"Oracle Corporation",
[junit4] 2> "cores":1,
[junit4] 2> "host":"127.0.0.1",
[junit4] 2> "freedisk":999,
[junit4] 2> "sysprop.java.version":"1.8.0_172",
[junit4] 2> "ip_1":"127",
[junit4] 2> "ip_3":"0",
[junit4] 2> "ip_2":"0",
[junit4] 2> "ip_4":"1",
[junit4] 2> "node":"127.0.0.1:10000_solr",
[junit4] 2> "metrics:solr.jvm:buffers.direct.Count":0,
[junit4] 2> "port":10000,
[junit4] 2> "heapUsage":123450000,
[junit4] 2> "sysLoadAvg":1.0,
[junit4] 2> "metrics:solr.node:ADMIN./admin/authorization.clientErrors:count":0},
[junit4] 2> "127.0.0.1:10002_solr":{
[junit4] 2> "sysprop.java.vendor":"Oracle Corporation",
[junit4] 2> "cores":1,
[junit4] 2> "host":"127.0.0.1",
[junit4] 2> "freedisk":1023,
[junit4] 2> "sysprop.java.version":"1.8.0_172",
[junit4] 2> "ip_1":"127",
[junit4] 2> "ip_3":"0",
[junit4] 2> "ip_2":"0",
[junit4] 2> "ip_4":"1",
[junit4] 2> "node":"127.0.0.1:10002_solr",
[junit4] 2> "metrics:solr.jvm:buffers.direct.Count":0,
[junit4] 2> "port":10002,
[junit4] 2> "heapUsage":123450000,
[junit4] 2> "sysLoadAvg":1.0,
[junit4] 2> "metrics:solr.node:ADMIN./admin/authorization.clientErrors:count":0}}
[junit4] 2> 1183626 INFO (TEST-TestSimComputePlanAction.testNodeLost-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.s.TestSimComputePlanAction * Live nodes: [127.0.0.1:10000_solr]
[junit4] 2> 1183627 INFO (TEST-TestSimComputePlanAction.testNodeLost-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.s.TestSimComputePlanAction * Collection .system state: DocCollection(.system//clusterstate.json/13)={
[junit4] 2> "replicationFactor":"1",
[junit4] 2> "pullReplicas":"0",
[junit4] 2> "router":{"name":"compositeId"},
[junit4] 2> "maxShardsPerNode":"1",
[junit4] 2> "autoAddReplicas":"false",
[junit4] 2> "nrtReplicas":"1",
[junit4] 2> "tlogReplicas":"0",
[junit4] 2> "autoCreated":"true",
[junit4] 2> "shards":{"shard1":{
[junit4] 2> "replicas":{"core_node1":{
[junit4] 2> "core":".system_shard1_replica_n1",
[junit4] 2> "SEARCHER.searcher.maxDoc":2,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":11240,
[junit4] 2> "node_name":"127.0.0.1:10002_solr",
[junit4] 2> "state":"down",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":1.0468065738677979E-5,
[junit4] 2> "SEARCHER.searcher.numDocs":2}},
[junit4] 2> "range":"80000000-7fffffff",
[junit4] 2> "state":"active"}}}
[junit4] 2> 1183627 INFO (TEST-TestSimComputePlanAction.testNodeLost-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.s.TestSimComputePlanAction * Collection testNodeLost state: DocCollection(testNodeLost//clusterstate.json/13)={
[junit4] 2> "replicationFactor":"1",
[junit4] 2> "pullReplicas":"0",
[junit4] 2> "router":{"name":"compositeId"},
[junit4] 2> "maxShardsPerNode":"1",
[junit4] 2> "autoAddReplicas":"false",
[junit4] 2> "nrtReplicas":"2",
[junit4] 2> "tlogReplicas":"0",
[junit4] 2> "autoCreated":"true",
[junit4] 2> "shards":{"shard1":{
[junit4] 2> "replicas":{
[junit4] 2> "core_node1":{
[junit4] 2> "core":"testNodeLost_shard1_replica_n1",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10001_solr",
[junit4] 2> "state":"down",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0},
[junit4] 2> "core_node2":{
[junit4] 2> "core":"testNodeLost_shard1_replica_n2",
[junit4] 2> "leader":"true",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10000_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0}},
[junit4] 2> "range":"80000000-7fffffff",
[junit4] 2> "state":"active"}}}
[junit4] 2> 1183628 INFO (TEST-TestSimComputePlanAction.testNodeLost-seed#[66FC441568EA3EF3]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testNodeLost
[junit4] 2> 1183629 DEBUG (simCloudManagerPool-4644-thread-13) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election for .system / shard1
[junit4] 2> 1183630 WARN (simCloudManagerPool-4644-thread-13) [ ] o.a.s.c.a.s.SimClusterStateProvider Can't find any active replicas for .system / shard1: shard1:{
[junit4] 2> "replicas":{"core_node1":{
[junit4] 2> "core":".system_shard1_replica_n1",
[junit4] 2> "SEARCHER.searcher.maxDoc":2,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":11240,
[junit4] 2> "node_name":"127.0.0.1:10002_solr",
[junit4] 2> "state":"down",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":1.0468065738677979E-5,
[junit4] 2> "SEARCHER.searcher.numDocs":2}},
[junit4] 2> "range":"80000000-7fffffff",
[junit4] 2> "state":"active"}
[junit4] 2> 1183630 DEBUG (simCloudManagerPool-4644-thread-13) [ ] o.a.s.c.a.s.SimClusterStateProvider -- liveNodes: [127.0.0.1:10000_solr]
[junit4] 2> 1183630 DEBUG (simCloudManagerPool-4644-thread-14) [ ] o.a.s.c.a.s.SimClusterStateProvider -- no leader in shard1:{
[junit4] 2> "replicas":{"core_node1":{
[junit4] 2> "core":".system_shard1_replica_n1",
[junit4] 2> "SEARCHER.searcher.maxDoc":2,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":11240,
[junit4] 2> "node_name":"127.0.0.1:10002_solr",
[junit4] 2> "state":"down",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":1.0468065738677979E-5,
[junit4] 2> "SEARCHER.searcher.numDocs":2}},
[junit4] 2> "range":"80000000-7fffffff",
[junit4] 2> "state":"active"}
[junit4] 2> 1183631 DEBUG (simCloudManagerPool-4644-thread-15) [ ] o.a.s.c.a.s.SimClusterStateProvider -- no leader in shard1:{
[junit4] 2> "replicas":{"core_node1":{
[junit4] 2> "core":".system_shard1_replica_n1",
[junit4] 2> "SEARCHER.searcher.maxDoc":2,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":11240,
[junit4] 2> "node_name":"127.0.0.1:10002_solr",
[junit4] 2> "state":"down",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":1.0468065738677979E-5,
[junit4] 2> "SEARCHER.searcher.numDocs":2}},
[junit4] 2> "range":"80000000-7fffffff",
[junit4] 2> "state":"active"}
[junit4] 2> 1183631 DEBUG (AutoscalingActionExecutor-4646-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms
[junit4] 2> 1183631 DEBUG (AutoscalingActionExecutor-4646-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .scheduled_maintenance after 100ms
[junit4] 2> 1183631 DEBUG (AutoscalingActionExecutor-4646-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_lost_trigger after 100ms
[junit4] 2> 1183631 DEBUG (AutoscalingActionExecutor-4646-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 985 ms for event id=eb5bf68a9c73cTw8ua0pe7uz880lqx2hc4v3nh
[junit4] 2> 1183633 INFO (TEST-TestSimComputePlanAction.testNodeLost-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase #######################################
[junit4] 2> ############ CLUSTER STATE ############
[junit4] 2> #######################################
[junit4] 2> ## Live nodes: 1
[junit4] 2> ## Empty nodes: 0
[junit4] 2> ## Dead nodes: 2
[junit4] 2> ## 127.0.0.1:10001_solr
[junit4] 2> ## 127.0.0.1:10002_solr
[junit4] 2> ## Collections: [.system, testNodeLost]
[junit4] 2> ## Max replicas per node: 1
[junit4] 2> ## Min replicas per node: 1
[junit4] 2> ## Total replicas: 1
[junit4] 2> ## * testNodeLost 1
[junit4] 2> ## - active 1
[junit4] 2> ######### Solr op counts ##########
[junit4] 2> ## - CREATE 1
[junit4] 2> ## - autoscaling 3
[junit4] 2> ## - update 4
[junit4] 2> ######### Autoscaling event counts ###########
[junit4] 2> ## * Trigger: node_lost_trigger
[junit4] 2> ## - AFTER_ACTION 1
[junit4] 2> ## - BEFORE_ACTION 1
[junit4] 2> ## - STARTED 1
[junit4] 2> ## - SUCCEEDED 1
[junit4] 2>
[junit4] 2> 1183650 INFO (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[66FC441568EA3EF3]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testNodeWithMultipleReplicasLost
[junit4] 2> 1183650 DEBUG (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 6
[junit4] 2> 1183650 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 6
[junit4] 2> 1183651 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
[junit4] 2> 1183651 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- deleted /autoscaling/nodeLost/127.0.0.1:10002_solr
[junit4] 2> 1183651 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 1183651 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion 6
[junit4] 2> 1183651 DEBUG (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.s.SimNodeStateProvider - removing dead node values: 127.0.0.1:10001_solr
[junit4] 2> 1183651 DEBUG (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.s.SimNodeStateProvider - removing dead node values: 127.0.0.1:10002_solr
[junit4] 2> 1183651 INFO (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
[junit4] 2> 1183651 DEBUG (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
[junit4] 2> 1183651 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 1183651 DEBUG (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now
[junit4] 2> 1183651 DEBUG (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
[junit4] 2> 1183651 DEBUG (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor
[junit4] 2> 1183651 DEBUG (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
[junit4] 2> 1183651 DEBUG (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
[junit4] 2> 1183657 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
[junit4] 2> 1183657 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 7
[junit4] 2> 1183658 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 7, lastZnodeVersion -1
[junit4] 2> 1183658 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 7
[junit4] 2> 1183659 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10000_solr]
[junit4] 2> 1183662 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 1183662 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 7, lastZnodeVersion 7
[junit4] 2> 1183662 DEBUG (ScheduledTrigger-4648-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 1183683 DEBUG (ScheduledTrigger-4648-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 1183703 DEBUG (ScheduledTrigger-4648-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 1183723 DEBUG (ScheduledTrigger-4648-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 1183744 DEBUG (ScheduledTrigger-4648-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 1183758 DEBUG (simCloudManagerPool-4647-thread-1) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 1183758 DEBUG (simCloudManagerPool-4647-thread-1) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 8
[junit4] 2> 1183759 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 8
[junit4] 2> 1183759 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 1183759 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 8, lastZnodeVersion 8
[junit4] 2> 1183761 DEBUG (simCloudManagerPool-4647-thread-2) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 1183761 DEBUG (simCloudManagerPool-4647-thread-2) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 9
[junit4] 2> 1183761 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 9
[junit4] 2> 1183762 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 1183762 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 9, lastZnodeVersion 9
[junit4] 2> 1183764 DEBUG (ScheduledTrigger-4648-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 1183784 DEBUG (ScheduledTrigger-4648-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 1183804 DEBUG (ScheduledTrigger-4648-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 1183825 DEBUG (ScheduledTrigger-4648-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 1183845 DEBUG (ScheduledTrigger-4648-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 1183864 DEBUG (simCloudManagerPool-4647-thread-3) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 1183864 DEBUG (simCloudManagerPool-4647-thread-3) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 10
[junit4] 2> 1183864 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 10
[junit4] 2> 1183865 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger node_lost_trigger - Initial livenodes: [127.0.0.1:10004_solr, 127.0.0.1:10005_solr, 127.0.0.1:10000_solr, 127.0.0.1:10003_solr]
[junit4] 2> 1183865 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 1183865 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 10, lastZnodeVersion 10
[junit4] 2> 1183865 DEBUG (ScheduledTrigger-4648-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 4
[junit4] 2> 1183865 DEBUG (ScheduledTrigger-4648-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
[junit4] 2> 1183865 DEBUG (simCloudManagerPool-4647-thread-4) [ ] o.a.s.c.o.ClusterStateMutator building a new cName: testNodeWithMultipleReplicasLost
[junit4] 2> 1183876 DEBUG (simCloudManagerPool-4647-thread-4) [ ] o.a.s.c.s.c.a.PolicyHelper Creating a new session
[junit4] 2> 1183876 DEBUG (simCloudManagerPool-4647-thread-4) [ ] o.a.s.c.s.c.a.PolicyHelper New session created
[junit4] 2> 1183881 INFO (simCloudManagerPool-4647-thread-4) [ ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 4140505701 sessionWrapper.createTime 4140505465621428, this.sessionWrapper.createTime 4140505465621428
[junit4] 2> 1183881 DEBUG (simCloudManagerPool-4647-thread-4) [ ] o.a.s.c.s.c.a.PolicyHelper session set to NULL
[junit4] 2> 1183886 DEBUG (ScheduledTrigger-4648-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 4
[junit4] 2> 1183886 DEBUG (ScheduledTrigger-4648-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
[junit4] 2> 1183892 DEBUG (simCloudManagerPool-4647-thread-5) [ ] o.a.s.c.a.s.SimClusterStateProvider -- no replicas in testNodeWithMultipleReplicasLost / shard2
[junit4] 2> 1183892 DEBUG (simCloudManagerPool-4647-thread-5) [ ] o.a.s.c.a.s.SimClusterStateProvider -- submit leader election for testNodeWithMultipleReplicasLost / shard1
[junit4] 2> 1183892 DEBUG (simCloudManagerPool-4647-thread-11) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election for testNodeWithMultipleReplicasLost / shard1
[junit4] 2> 1183893 DEBUG (simCloudManagerPool-4647-thread-11) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeWithMultipleReplicasLost / shard1: core_node1
[junit4] 2> 1183903 DEBUG (simCloudManagerPool-4647-thread-6) [ ] o.a.s.c.a.s.SimClusterStateProvider -- no replicas in testNodeWithMultipleReplicasLost / shard2
[junit4] 2> 1183903 DEBUG (simCloudManagerPool-4647-thread-6) [ ] o.a.s.c.a.s.SimClusterStateProvider -- already has leader testNodeWithMultipleReplicasLost / shard1
[junit4] 2> 1183906 DEBUG (ScheduledTrigger-4648-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 4
[junit4] 2> 1183906 DEBUG (ScheduledTrigger-4648-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
[junit4] 2> 1183914 DEBUG (simCloudManagerPool-4647-thread-7) [ ] o.a.s.c.a.s.SimClusterStateProvider -- no replicas in testNodeWithMultipleReplicasLost / shard2
[junit4] 2> 1183914 DEBUG (simCloudManagerPool-4647-thread-7) [ ] o.a.s.c.a.s.SimClusterStateProvider -- already has leader testNodeWithMultipleReplicasLost / shard1
[junit4] 2> 1183924 DEBUG (simCloudManagerPool-4647-thread-8) [ ] o.a.s.c.a.s.SimClusterStateProvider -- submit leader election for testNodeWithMultipleReplicasLost / shard2
[junit4] 2> 1183925 DEBUG (simCloudManagerPool-4647-thread-8) [ ] o.a.s.c.a.s.SimClusterStateProvider -- already has leader testNodeWithMultipleReplicasLost / shard1
[junit4] 2> 1183925 DEBUG (simCloudManagerPool-4647-thread-12) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election for testNodeWithMultipleReplicasLost / shard2
[junit4] 2> 1183925 DEBUG (simCloudManagerPool-4647-thread-12) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeWithMultipleReplicasLost / shard2: core_node4
[junit4] 2> 1183926 DEBUG (ScheduledTrigger-4648-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 4
[junit4] 2> 1183926 DEBUG (ScheduledTrigger-4648-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
[junit4] 2> 1183935 DEBUG (simCloudManagerPool-4647-thread-9) [ ] o.a.s.c.a.s.SimClusterStateProvider -- already has leader testNodeWithMultipleReplicasLost / shard2
[junit4] 2> 1183935 DEBUG (simCloudManagerPool-4647-thread-9) [ ] o.a.s.c.a.s.SimClusterStateProvider -- already has leader testNodeWithMultipleReplicasLost / shard1
[junit4] 2> 1183946 DEBUG (simCloudManagerPool-4647-thread-10) [ ] o.a.s.c.a.s.SimClusterStateProvider -- already has leader testNodeWithMultipleReplicasLost / shard2
[junit4] 2> 1183946 DEBUG (simCloudManagerPool-4647-thread-10) [ ] o.a.s.c.a.s.SimClusterStateProvider -- already has leader testNodeWithMultipleReplicasLost / shard1
[junit4] 2> 1183946 DEBUG (ScheduledTrigger-4648-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 4
[junit4] 2> 1183946 DEBUG (ScheduledTrigger-4648-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
[junit4] 2> 1183947 DEBUG (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.s.TestSimComputePlanAction -- cluster state: znodeVersion: 25
[junit4] 2> live nodes:[127.0.0.1:10004_solr, 127.0.0.1:10000_solr, 127.0.0.1:10003_solr, 127.0.0.1:10005_solr]
[junit4] 2> collections:{testNodeWithMultipleReplicasLost=DocCollection(testNodeWithMultipleReplicasLost//clusterstate.json/24)={
[junit4] 2> "replicationFactor":"1",
[junit4] 2> "pullReplicas":"0",
[junit4] 2> "router":{"name":"compositeId"},
[junit4] 2> "maxShardsPerNode":"1",
[junit4] 2> "autoAddReplicas":"false",
[junit4] 2> "nrtReplicas":"3",
[junit4] 2> "tlogReplicas":"0",
[junit4] 2> "autoCreated":"true",
[junit4] 2> "shards":{
[junit4] 2> "shard2":{
[junit4] 2> "replicas":{
[junit4] 2> "core_node6":{
[junit4] 2> "core":"testNodeWithMultipleReplicasLost_shard2_replica_n6",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10005_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0},
[junit4] 2> "core_node4":{
[junit4] 2> "core":"testNodeWithMultipleReplicasLost_shard2_replica_n4",
[junit4] 2> "leader":"true",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10000_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0},
[junit4] 2> "core_node5":{
[junit4] 2> "core":"testNodeWithMultipleReplicasLost_shard2_replica_n5",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10004_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0}},
[junit4] 2> "range":"0-7fffffff",
[junit4] 2> "state":"active"},
[junit4] 2> "shard1":{
[junit4] 2> "replicas":{
[junit4] 2> "core_node1":{
[junit4] 2> "core":"testNodeWithMultipleReplicasLost_shard1_replica_n1",
[junit4] 2> "leader":"true",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10003_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0},
[junit4] 2> "core_node2":{
[junit4] 2> "core":"testNodeWithMultipleReplicasLost_shard1_replica_n2",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10005_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0},
[junit4] 2> "core_node3":{
[junit4] 2> "core":"testNodeWithMultipleReplicasLost_shard1_replica_n3",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10004_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0}},
[junit4] 2> "range":"80000000-ffffffff",
[junit4] 2> "state":"active"}}}}
[junit4] 2> 1183948 DEBUG (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.s.SimClusterStateProvider -- already has leader testNodeWithMultipleReplicasLost / shard2
[junit4] 2> 1183948 DEBUG (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.s.SimClusterStateProvider -- already has leader testNodeWithMultipleReplicasLost / shard1
[junit4] 2> 1183967 DEBUG (ScheduledTrigger-4648-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 3
[junit4] 2> 1183967 DEBUG (ScheduledTrigger-4648-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
[junit4] 2> 1183967 DEBUG (ScheduledTrigger-4648-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10004_solr
[junit4] 2> 1183967 DEBUG (ScheduledTrigger-4648-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10004_solr
[junit4] 2> 1183987 DEBUG (ScheduledTrigger-4648-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
[junit4] 2> 1183987 DEBUG (ScheduledTrigger-4648-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 3
[junit4] 2> 1183987 DEBUG (ScheduledTrigger-4648-thread-4) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:10004_solr]
[junit4] 2> 1183987 DEBUG (ScheduledTrigger-4648-thread-4) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
[junit4] 2> "id":"eb5c59ade8516Tw8ua0pe7uz880lqx2hc4v3nm",
[junit4] 2> "source":"node_lost_trigger",
[junit4] 2> "eventTime":4140509985408278,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[4140509985408278],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "nodeNames":["127.0.0.1:10004_solr"]}}
[junit4] 2> 1183988 DEBUG (ScheduledTrigger-4648-thread-4) [ ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, .scheduled_maintenance, node_lost_trigger]
[junit4] 2> 1184004 DEBUG (simCloudManagerPool-4647-thread-13) [ ] o.a.s.c.o.ClusterStateMutator building a new cName: .system
[junit4] 2> 1184017 DEBUG (simCloudManagerPool-4647-thread-13) [ ] o.a.s.c.s.c.a.PolicyHelper Creating a new session
[junit4] 2> 1184018 DEBUG (simCloudManagerPool-4647-thread-13) [ ] o.a.s.c.s.c.a.PolicyHelper New session created
[junit4] 2> 1184025 INFO (simCloudManagerPool-4647-thread-13) [ ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 4140512900 sessionWrapper.createTime 4140512536873878, this.sessionWrapper.createTime 4140512536873878
[junit4] 2> 1184025 DEBUG (simCloudManagerPool-4647-thread-13) [ ] o.a.s.c.s.c.a.PolicyHelper session set to NULL
[junit4] 2> 1184036 DEBUG (simCloudManagerPool-4647-thread-14) [ ] o.a.s.c.a.s.SimClusterStateProvider -- submit leader election for .system / shard1
[junit4] 2> 1184036 DEBUG (simCloudManagerPool-4647-thread-15) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election for .system / shard1
[junit4] 2> 1184036 DEBUG (simCloudManagerPool-4647-thread-15) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for .system / shard1: core_node2
[junit4] 2> 1184038 DEBUG (AutoscalingActionExecutor-4649-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
[junit4] 2> "id":"eb5c59ade8516Tw8ua0pe7uz880lqx2hc4v3nm",
[junit4] 2> "source":"node_lost_trigger",
[junit4] 2> "eventTime":4140509985408278,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[4140509985408278],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":4140511034092428,
[junit4] 2> "nodeNames":["127.0.0.1:10004_solr"]}}
[junit4] 2> 1184038 DEBUG (AutoscalingActionExecutor-4649-thread-1) [ ] o.a.s.c.a.ComputePlanAction -- processing event: {
[junit4] 2> "id":"eb5c59ade8516Tw8ua0pe7uz880lqx2hc4v3nm",
[junit4] 2> "source":"node_lost_trigger",
[junit4] 2> "eventTime":4140509985408278,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[4140509985408278],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":4140511034092428,
[junit4] 2> "nodeNames":["127.0.0.1:10004_solr"]}} with context properties: {BEFORE_ACTION=[compute_plan]}
[junit4] 2> 1184038 DEBUG (AutoscalingActionExecutor-4649-thread-1) [ ] o.a.s.c.s.c.a.PolicyHelper Creating a new session
[junit4] 2> 1184039 DEBUG (AutoscalingActionExecutor-4649-thread-1) [ ] o.a.s.c.s.c.a.PolicyHelper New session created
[junit4] 2> 1184041 INFO (AutoscalingActionExecutor-4649-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testNodeWithMultipleReplicasLost&targetNode=127.0.0.1:10003_solr&inPlaceMove=true&replica=core_node5
[junit4] 2> 1184041 INFO (AutoscalingActionExecutor-4649-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testNodeWithMultipleReplicasLost&targetNode=127.0.0.1:10000_solr&inPlaceMove=true&replica=core_node3
[junit4] 2> 1184042 INFO (AutoscalingActionExecutor-4649-thread-1) [ ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 4140513733 sessionWrapper.createTime 4140513604996028, this.sessionWrapper.createTime 4140513604996028
[junit4] 2> 1184042 DEBUG (AutoscalingActionExecutor-4649-thread-1) [ ] o.a.s.c.s.c.a.PolicyHelper session set to NULL
[junit4] 2> 1184043 INFO (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.s.TestSimComputePlanAction action=MOVEREPLICA&collection=testNodeWithMultipleReplicasLost&targetNode=127.0.0.1:10003_solr&inPlaceMove=true&replica=core_node5
[junit4] 2> 1184043 INFO (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.s.TestSimComputePlanAction action=MOVEREPLICA&collection=testNodeWithMultipleReplicasLost&targetNode=127.0.0.1:10000_solr&inPlaceMove=true&replica=core_node3
[junit4] 2> 1184043 INFO (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.s.TestSimComputePlanAction -------------_ FINAL STATE --------------
[junit4] 2> 1184043 INFO (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.s.TestSimComputePlanAction * Node values: {
[junit4] 2> "127.0.0.1:10004_solr":{
[junit4] 2> "sysprop.java.vendor":"Oracle Corporation",
[junit4] 2> "cores":2,
[junit4] 2> "host":"127.0.0.1",
[junit4] 2> "freedisk":1022,
[junit4] 2> "sysprop.java.version":"1.8.0_172",
[junit4] 2> "ip_1":"127",
[junit4] 2> "ip_3":"0",
[junit4] 2> "ip_2":"0",
[junit4] 2> "ip_4":"1",
[junit4] 2> "node":"127.0.0.1:10004_solr",
[junit4] 2> "metrics:solr.jvm:buffers.direct.Count":0,
[junit4] 2> "port":10004,
[junit4] 2> "heapUsage":123450000,
[junit4] 2> "sysLoadAvg":1.0,
[junit4] 2> "metrics:solr.node:ADMIN./admin/authorization.clientErrors:count":0},
[junit4] 2> "127.0.0.1:10005_solr":{
[junit4] 2> "sysprop.java.vendor":"Oracle Corporation",
[junit4] 2> "cores":2,
[junit4] 2> "host":"127.0.0.1",
[junit4] 2> "freedisk":1022,
[junit4] 2> "sysprop.java.version":"1.8.0_172",
[junit4] 2> "ip_1":"127",
[junit4] 2> "ip_3":"0",
[junit4] 2> "ip_2":"0",
[junit4] 2> "ip_4":"1",
[junit4] 2> "node":"127.0.0.1:10005_solr",
[junit4] 2> "metrics:solr.jvm:buffers.direct.Count":0,
[junit4] 2> "port":10005,
[junit4] 2> "heapUsage":123450000,
[junit4] 2> "sysLoadAvg":1.0,
[junit4] 2> "metrics:solr.node:ADMIN./admin/authorization.clientErrors:count":0},
[junit4] 2> "127.0.0.1:10000_solr":{
[junit4] 2> "sysprop.java.vendor":"Oracle Corporation",
[junit4] 2> "cores":1,
[junit4] 2> "host":"127.0.0.1",
[junit4] 2> "freedisk":999,
[junit4] 2> "sysprop.java.version":"1.8.0_172",
[junit4] 2> "ip_1":"127",
[junit4] 2> "ip_3":"0",
[junit4] 2> "ip_2":"0",
[junit4] 2> "ip_4":"1",
[junit4] 2> "node":"127.0.0.1:10000_solr",
[junit4] 2> "metrics:solr.jvm:buffers.direct.Count":0,
[junit4] 2> "port":10000,
[junit4] 2> "heapUsage":123450000,
[junit4] 2> "sysLoadAvg":1.0,
[junit4] 2> "metrics:solr.node:ADMIN./admin/authorization.clientErrors:count":0},
[junit4] 2> "127.0.0.1:10003_solr":{
[junit4] 2> "sysprop.java.vendor":"Oracle Corporation",
[junit4] 2> "cores":2,
[junit4] 2> "host":"127.0.0.1",
[junit4] 2> "freedisk":1022,
[junit4] 2> "sysprop.java.version":"1.8.0_172",
[junit4] 2> "ip_1":"127",
[junit4] 2> "ip_3":"0",
[junit4] 2> "ip_2":"0",
[junit4] 2> "ip_4":"1",
[junit4] 2> "node":"127.0.0.1:10003_solr",
[junit4] 2> "metrics:solr.jvm:buffers.direct.Count":0,
[junit4] 2> "port":10003,
[junit4] 2> "heapUsage":123450000,
[junit4] 2> "sysLoadAvg":1.0,
[junit4] 2> "metrics:solr.node:ADMIN./admin/authorization.clientErrors:count":0}}
[junit4] 2> 1184043 INFO (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.s.TestSimComputePlanAction * Live nodes: [127.0.0.1:10005_solr, 127.0.0.1:10000_solr, 127.0.0.1:10003_solr]
[junit4] 2> 1184044 INFO (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.s.TestSimComputePlanAction * Collection .system state: DocCollection(.system//clusterstate.json/32)={
[junit4] 2> "replicationFactor":"1",
[junit4] 2> "pullReplicas":"0",
[junit4] 2> "router":{"name":"compositeId"},
[junit4] 2> "maxShardsPerNode":"1",
[junit4] 2> "autoAddReplicas":"false",
[junit4] 2> "nrtReplicas":"1",
[junit4] 2> "tlogReplicas":"0",
[junit4] 2> "autoCreated":"true",
[junit4] 2> "shards":{"shard1":{
[junit4] 2> "replicas":{"core_node2":{
[junit4] 2> "core":".system_shard1_replica_n1",
[junit4] 2> "leader":"true",
[junit4] 2> "SEARCHER.searcher.maxDoc":2,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":11240,
[junit4] 2> "node_name":"127.0.0.1:10003_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":1.0468065738677979E-5,
[junit4] 2> "SEARCHER.searcher.numDocs":2}},
[junit4] 2> "range":"80000000-7fffffff",
[junit4] 2> "state":"active"}}}
[junit4] 2> 1184044 INFO (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.s.TestSimComputePlanAction * Collection testNodeWithMultipleReplicasLost state: DocCollection(testNodeWithMultipleReplicasLost//clusterstate.json/32)={
[junit4] 2> "replicationFactor":"1",
[junit4] 2> "pullReplicas":"0",
[junit4] 2> "router":{"name":"compositeId"},
[junit4] 2> "maxShardsPerNode":"1",
[junit4] 2> "autoAddReplicas":"false",
[junit4] 2> "nrtReplicas":"3",
[junit4] 2> "tlogReplicas":"0",
[junit4] 2> "autoCreated":"true",
[junit4] 2> "shards":{
[junit4] 2> "shard2":{
[junit4] 2> "replicas":{
[junit4] 2> "core_node6":{
[junit4] 2> "core":"testNodeWithMultipleReplicasLost_shard2_replica_n6",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10005_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0},
[junit4] 2> "core_node4":{
[junit4] 2> "core":"testNodeWithMultipleReplicasLost_shard2_replica_n4",
[junit4] 2> "leader":"true",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10000_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0},
[junit4] 2> "core_node5":{
[junit4] 2> "core":"testNodeWithMultipleReplicasLost_shard2_replica_n5",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10004_solr",
[junit4] 2> "state":"down",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0}},
[junit4] 2> "range":"0-7fffffff",
[junit4] 2> "state":"active"},
[junit4] 2> "shard1":{
[junit4] 2> "replicas":{
[junit4] 2> "core_node1":{
[junit4] 2> "core":"testNodeWithMultipleReplicasLost_shard1_replica_n1",
[junit4] 2> "leader":"true",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10003_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0},
[junit4] 2> "core_node2":{
[junit4] 2> "core":"testNodeWithMultipleReplicasLost_shard1_replica_n2",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10005_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0},
[junit4] 2> "core_node3":{
[junit4] 2> "core":"testNodeWithMultipleReplicasLost_shard1_replica_n3",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10004_solr",
[junit4] 2> "state":"down",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0}},
[junit4] 2> "range":"80000000-ffffffff",
[junit4] 2> "state":"active"}}}
[junit4] 2> 1184045 INFO (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[66FC441568EA3EF3]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testNodeWithMultipleReplicasLost
[junit4] 2> 1184045 INFO (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase #######################################
[junit4] 2> ############ CLUSTER STATE ############
[junit4] 2> #######################################
[junit4] 2> ## Live nodes: 3
[junit4] 2> ## Empty nodes: 0
[junit4] 2> ## Dead nodes: 1
[junit4] 2> ## 127.0.0.1:10004_solr
[junit4] 2> ## Collections: [.system, testNodeWithMultipleReplicasLost]
[junit4] 2> ## Max replicas per node: 2
[junit4] 2> ## Min replicas per node: 1
[junit4] 2> ## Total replicas: 5
[junit4] 2> ## * .system 1
[junit4] 2> ## - active 1
[junit4] 2> ## * testNodeWithMultipleReplicasLost 4
[junit4] 2> ## - active 4
[junit4] 2> ######### Solr op counts ##########
[junit4] 2> ## - CREATE 1
[junit4] 2> ## - autoscaling 3
[junit4] 2> ## - update 2
[junit4] 2> ######### Autoscaling event counts ###########
[junit4] 2> ## * Trigger: node_lost_trigger
[junit4] 2> ## - BEFORE_ACTION 1
[junit4] 2> ## - STARTED 1
[junit4] 2>
[junit4] 2> 1184058 INFO (TEST-TestSimComputePlanAction.testNodeAdded-seed#[66FC441568EA3EF3]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testNodeAdded
[junit4] 2> 1184058 DEBUG (TEST-TestSimComputePlanAction.testNodeAdded-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 11
[junit4] 2> 1184058 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 11
[junit4] 2> 1184058 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
[junit4] 2> 1184058 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 1184058 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 11, lastZnodeVersion 11
[junit4] 2> 1184059 DEBUG (TEST-TestSimComputePlanAction.testNodeAdded-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.s.SimNodeStateProvider - removing dead node values: 127.0.0.1:10004_solr
[junit4] 2> 1184059 DEBUG (TEST-TestSimComputePlanAction.testNodeAdded-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.s.SimClusterStateProvider --- new Overseer leader: 127.0.0.1:10005_solr
[junit4] 2> 1184059 DEBUG (TEST-TestSimComputePlanAction.testNodeAdded-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.s.SimNodeStateProvider --removing value for 127.0.0.1:10000_solr
[junit4] 2> 1184059 DEBUG (TEST-TestSimComputePlanAction.testNodeAdded-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.s.SimClusterStateProvider --- new Overseer leader: 127.0.0.1:10003_solr
[junit4] 2> 1184059 DEBUG (TEST-TestSimComputePlanAction.testNodeAdded-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.s.SimNodeStateProvider --removing value for 127.0.0.1:10005_solr
[junit4] 2> 1184059 INFO (TEST-TestSimComputePlanAction.testNodeAdded-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
[junit4] 2> 1184059 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Interrupted
[junit4] 2> java.lang.InterruptedException: null
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.reportInterruptAfterWait(AbstractQueuedSynchronizer.java:2014) ~[?:1.8.0_172]
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2048) ~[?:1.8.0_172]
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:189) [java/:?]
[junit4] 2> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_172]
[junit4] 2> 1184059 DEBUG (TEST-TestSimComputePlanAction.testNodeAdded-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
[junit4] 2> 1184060 DEBUG (TEST-TestSimComputePlanAction.testNodeAdded-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now
[junit4] 2> 1184060 DEBUG (TEST-TestSimComputePlanAction.testNodeAdded-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
[junit4] 2> 1184060 DEBUG (simCloudManagerPool-4647-thread-17) [ ] o.a.s.c.o.ClusterStateMutator building a new cName: .system
[junit4] 2> 1184060 WARN (AutoscalingActionExecutor-4649-thread-1) [ ] o.a.s.c.a.SystemLogListener Exception sending event to collection .system
[junit4] 2> java.io.IOException: java.lang.InterruptedException
[junit4] 2> at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.request(SimCloudManager.java:632) ~[test/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.SystemLogListener.onEvent(SystemLogListener.java:118) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers$TriggerListeners.fireListeners(ScheduledTriggers.java:792) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers$TriggerListeners.fireListeners(ScheduledTriggers.java:768) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:315) ~[java/:?]
[junit4] 2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_172]
[junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_172]
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) ~[java/:?]
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_172]
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_172]
[junit4] 2> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_172]
[junit4] 2> Caused by: java.lang.InterruptedException
[junit4] 2> at java.util.concurrent.FutureTask.awaitDone(FutureTask.java:404) ~[?:1.8.0_172]
[junit4] 2> at java.util.concurrent.FutureTask.get(FutureTask.java:191) ~[?:1.8.0_172]
[junit4] 2> at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.request(SimCloudManager.java:630) ~[test/:?]
[junit4] 2> ... 10 more
[junit4] 2> 1184062 DEBUG (simCloudManagerPool-4647-thread-18) [ ] o.a.s.c.o.ClusterStateMutator building a new cName: .system
[junit4] 2> 1184081 DEBUG (simCloudManagerPool-4647-thread-19) [ ] o.a.s.c.a.s.SimClusterStateProvider -- submit leader election for .system / shard1
[junit4] 2> 1184083 DEBUG (simCloudManagerPool-4647-thread-20) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election for .system / shard1
[junit4] 2> 1184083 DEBUG (simCloudManagerPool-4647-thread-20) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for .system / shard1: core_node3
[junit4] 2> 1185284 WARN (AutoscalingActionExecutor-4649-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Unhandled exception executing actions
[junit4] 2> org.apache.lucene.store.AlreadyClosedException: ScheduledTrigger node_lost_trigger has been closed.
[junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers$TriggerWrapper.dequeue(ScheduledTriggers.java:548) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:318) ~[java/:?]
[junit4] 2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_172]
[junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_172]
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) ~[java/:?]
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_172]
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_172]
[junit4] 2> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_172]
[junit4] 2> 1185284 DEBUG (AutoscalingActionExecutor-4649-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 62318 ms for event id=eb5c59ade8516Tw8ua0pe7uz880lqx2hc4v3nm
[junit4] 2> 1185284 DEBUG (TEST-TestSimComputePlanAction.testNodeAdded-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor
[junit4] 2> 1185284 DEBUG (TEST-TestSimComputePlanAction.testNodeAdded-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
[junit4] 2> 1185284 DEBUG (TEST-TestSimComputePlanAction.testNodeAdded-seed#[66FC441568EA3EF3]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
[junit4] 2> 1185285 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
[junit4] 2> 1185285 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 12
[junit4] 2> 1185286 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 12, lastZnodeVersion -1
[junit4] 2> 1185286 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 12
[junit4] 2> 1185286 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10003_solr]
[junit4] 2> 1185287 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 1185287 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 12, lastZnodeVersion 12
[junit4] 2> 1185287 DEBUG (ScheduledTrigger-4651-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 1185308 DEBUG (ScheduledTrigger-4651-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 1185328 DEBUG (ScheduledTrigger-4651-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 1185348 DEBUG (ScheduledTrigger-4651-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 1185368 DEBUG (ScheduledTrigger-4651-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 1185387 DEBUG (simCloudManagerPool-4650-thread-1) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 1185387 DEBUG (simCloudManagerPool-4650-thread-1) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 13
[junit4] 2> 1185389 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 13
[junit4] 2> 1185389 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 1185389 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 13, lastZnodeVersion 13
[junit4] 2> 1185389 DEBUG (ScheduledTrigger-4651-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigg
[...truncated too long message...]
ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
jar-checksums:
[mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/null1239440791
[copy] Copying 238 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/null1239440791
[delete] Deleting directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/null1239440791
check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] confs: [default]
[ivy:cachepath] found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 19ms :: artifacts dl 1ms
---------------------------------------------------------------------
| | modules || artifacts |
| conf | number| search|dwnlded|evicted|| number|dwnlded|
---------------------------------------------------------------------
| default | 8 | 0 | 0 | 0 || 8 | 0 |
---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...
-jenkins-base:
BUILD SUCCESSFUL
Total time: 99 minutes 50 seconds
Archiving artifacts
java.lang.InterruptedException: no matches found within 10000
at hudson.FilePath$34.hasMatch(FilePath.java:2678)
at hudson.FilePath$34.invoke(FilePath.java:2557)
at hudson.FilePath$34.invoke(FilePath.java:2547)
at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2918)
Also: hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
at hudson.remoting.Channel.call(Channel.java:955)
at hudson.FilePath.act(FilePath.java:1036)
at hudson.FilePath.act(FilePath.java:1025)
at hudson.FilePath.validateAntFileMask(FilePath.java:2547)
at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
at hudson.model.Build$BuildExecution.post2(Build.java:186)
at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
at hudson.model.Run.execute(Run.java:1819)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
Caused: hudson.FilePath$TunneledInterruptedException
at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2920)
at hudson.remoting.UserRequest.perform(UserRequest.java:212)
at hudson.remoting.UserRequest.perform(UserRequest.java:54)
at hudson.remoting.Request$2.run(Request.java:369)
at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
Caused: java.lang.InterruptedException: java.lang.InterruptedException: no matches found within 10000
at hudson.FilePath.act(FilePath.java:1038)
at hudson.FilePath.act(FilePath.java:1025)
at hudson.FilePath.validateAntFileMask(FilePath.java:2547)
at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
at hudson.model.Build$BuildExecution.post2(Build.java:186)
at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
at hudson.model.Run.execute(Run.java:1819)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)