You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2018/12/24 06:37:32 UTC

[JENKINS] Lucene-Solr-BadApples-7.x-Linux (64bit/jdk-9.0.4) - Build # 138 - Unstable!

Build: https://jenkins.thetaphi.de/job/Lucene-Solr-BadApples-7.x-Linux/138/
Java: 64bit/jdk-9.0.4 -XX:-UseCompressedOops -XX:+UseParallelGC

5 tests failed.
FAILED:  org.apache.solr.client.solrj.io.stream.StreamDecoratorTest.testParallelCommitStream

Error Message:
expected:<5> but was:<2>

Stack Trace:
java.lang.AssertionError: expected:<5> but was:<2>
	at __randomizedtesting.SeedInfo.seed([AF9391FC404617DC:8F79F3FCDC07FA90]:0)
	at org.junit.Assert.fail(Assert.java:88)
	at org.junit.Assert.failNotEquals(Assert.java:834)
	at org.junit.Assert.assertEquals(Assert.java:645)
	at org.junit.Assert.assertEquals(Assert.java:631)
	at org.apache.solr.client.solrj.io.stream.StreamDecoratorTest.testParallelCommitStream(StreamDecoratorTest.java:3034)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.base/java.lang.Thread.run(Thread.java:844)


FAILED:  org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration.testNodeMarkersRegistration

Error Message:
Path /autoscaling/nodeAdded/127.0.0.1:10055_solr should have been deleted

Stack Trace:
java.lang.AssertionError: Path /autoscaling/nodeAdded/127.0.0.1:10055_solr should have been deleted
	at __randomizedtesting.SeedInfo.seed([77FFAC40FA650924:6F45244CF450C4CB]:0)
	at org.junit.Assert.fail(Assert.java:88)
	at org.junit.Assert.assertTrue(Assert.java:41)
	at org.junit.Assert.assertFalse(Assert.java:64)
	at org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration.testNodeMarkersRegistration(TestSimTriggerIntegration.java:891)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.base/java.lang.Thread.run(Thread.java:844)


FAILED:  org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration.testNodeMarkersRegistration

Error Message:
Path /autoscaling/nodeAdded/127.0.0.1:10055_solr should have been deleted

Stack Trace:
java.lang.AssertionError: Path /autoscaling/nodeAdded/127.0.0.1:10055_solr should have been deleted
	at __randomizedtesting.SeedInfo.seed([77FFAC40FA650924:6F45244CF450C4CB]:0)
	at org.junit.Assert.fail(Assert.java:88)
	at org.junit.Assert.assertTrue(Assert.java:41)
	at org.junit.Assert.assertFalse(Assert.java:64)
	at org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration.testNodeMarkersRegistration(TestSimTriggerIntegration.java:891)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.base/java.lang.Thread.run(Thread.java:844)


FAILED:  org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration.testNodeAddedTriggerRestoreState

Error Message:
The trigger did not fire at all

Stack Trace:
java.lang.AssertionError: The trigger did not fire at all
	at __randomizedtesting.SeedInfo.seed([77FFAC40FA650924:FFC2253FC0A5E889]:0)
	at org.junit.Assert.fail(Assert.java:88)
	at org.junit.Assert.assertTrue(Assert.java:41)
	at org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration.testNodeAddedTriggerRestoreState(TestSimTriggerIntegration.java:390)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.base/java.lang.Thread.run(Thread.java:844)


FAILED:  org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration.testNodeMarkersRegistration

Error Message:
Path /autoscaling/nodeAdded/127.0.0.1:10304_solr should have been deleted

Stack Trace:
java.lang.AssertionError: Path /autoscaling/nodeAdded/127.0.0.1:10304_solr should have been deleted
	at __randomizedtesting.SeedInfo.seed([77FFAC40FA650924:6F45244CF450C4CB]:0)
	at org.junit.Assert.fail(Assert.java:88)
	at org.junit.Assert.assertTrue(Assert.java:41)
	at org.junit.Assert.assertFalse(Assert.java:64)
	at org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration.testNodeMarkersRegistration(TestSimTriggerIntegration.java:891)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.base/java.lang.Thread.run(Thread.java:844)




Build Log:
[...truncated 14570 lines...]
   [junit4] Suite: org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration
   [junit4]   2> Creating dataDir: /home/jenkins/workspace/Lucene-Solr-BadApples-7.x-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.autoscaling.sim.TestSimTriggerIntegration_77FFAC40FA650924-001/init-core-data-001
   [junit4]   2> 1663298 DEBUG (TEST-TestSimTriggerIntegration.testListeners-seed#[77FFAC40FA650924]) [    ] o.a.s.c.a.s.SimClusterStateProvider --- new Overseer leader: 127.0.0.1:10277_solr
   [junit4]   2> 1663298 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
   [junit4]   2> 1663298 DEBUG (TEST-TestSimTriggerIntegration.testListeners-seed#[77FFAC40FA650924]) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=0
   [junit4]   2> 1663298 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
   [junit4]   2> 1663298 DEBUG (TEST-TestSimTriggerIntegration.testListeners-seed#[77FFAC40FA650924]) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 0
   [junit4]   2> 1663298 INFO  (TEST-TestSimTriggerIntegration.testListeners-seed#[77FFAC40FA650924]) [    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 1663299 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 0
   [junit4]   2> 1663299 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
   [junit4]   2> 1663299 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10278_solr, 127.0.0.1:10277_solr]
   [junit4]   2> 1663299 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
   [junit4]   2> 1663299 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
   [junit4]   2> 1663299 DEBUG (ScheduledTrigger-11489-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
   [junit4]   2> 1663300 DEBUG (simCloudManagerPool-11488-thread-2) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1663300 DEBUG (simCloudManagerPool-11488-thread-2) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2
   [junit4]   2> 1663300 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
   [junit4]   2> 1663300 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
   [junit4]   2> 1663300 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
   [junit4]   2> 1663302 DEBUG (simCloudManagerPool-11488-thread-3) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1663302 DEBUG (simCloudManagerPool-11488-thread-3) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 3
   [junit4]   2> 1663302 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
   [junit4]   2> 1663302 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.TestSimTriggerIntegration TestTriggerAction init
   [junit4]   2> 1663302 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger - Initial livenodes: [127.0.0.1:10278_solr, 127.0.0.1:10277_solr]
   [junit4]   2> 1663302 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated with properties: {event=nodeAdded, waitFor=3, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration$TestTriggerAction}, {name=test1, class=org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration$TestDummyAction}], enabled=true}
   [junit4]   2> 1663302 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
   [junit4]   2> 1663302 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
   [junit4]   2> 1663302 DEBUG (ScheduledTrigger-11489-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1663302 DEBUG (ScheduledTrigger-11489-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
   [junit4]   2> 1663303 DEBUG (simCloudManagerPool-11488-thread-4) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1663303 DEBUG (simCloudManagerPool-11488-thread-4) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 4
   [junit4]   2> 1663303 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 4
   [junit4]   2> 1663304 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
   [junit4]   2> 1663304 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion 4
   [junit4]   2> 1663304 DEBUG (simCloudManagerPool-11488-thread-5) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1663304 DEBUG (simCloudManagerPool-11488-thread-5) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 5
   [junit4]   2> 1663304 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 5
   [junit4]   2> 1663304 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
   [junit4]   2> 1663304 DEBUG (TEST-TestSimTriggerIntegration.testListeners-seed#[77FFAC40FA650924]) [    ] o.a.s.c.a.s.SimClusterStateProvider -- creating marker: /autoscaling/nodeAdded/127.0.0.1:10279_solr
   [junit4]   2> 1663304 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 5, lastZnodeVersion 5
   [junit4]   2> 1663320 DEBUG (ScheduledTrigger-11489-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 2
   [junit4]   2> 1663323 DEBUG (ScheduledTrigger-11489-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1663323 DEBUG (ScheduledTrigger-11489-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1663323 DEBUG (ScheduledTrigger-11489-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10279_solr at time 615681769844594
   [junit4]   2> 1663340 DEBUG (ScheduledTrigger-11489-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
   [junit4]   2> 1663343 DEBUG (ScheduledTrigger-11489-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1663343 DEBUG (ScheduledTrigger-11489-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1663360 DEBUG (ScheduledTrigger-11489-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
   [junit4]   2> 1663363 DEBUG (ScheduledTrigger-11489-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1663363 DEBUG (ScheduledTrigger-11489-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1663380 DEBUG (ScheduledTrigger-11489-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
   [junit4]   2> 1663383 DEBUG (ScheduledTrigger-11489-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1663383 DEBUG (ScheduledTrigger-11489-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1663383 DEBUG (ScheduledTrigger-11489-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger firing registered processor for nodes: [127.0.0.1:10279_solr] added at times [615681769844594], now=615684803086194
   [junit4]   2> 1663384 DEBUG (simCloudManagerPool-11488-thread-6) [    ] o.a.s.c.a.s.SimClusterStateProvider -- simCreateCollection .system, currentVersion=1
   [junit4]   2> 1663384 DEBUG (simCloudManagerPool-11488-thread-6) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=1
   [junit4]   2> 1663384 DEBUG (simCloudManagerPool-11488-thread-6) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 1
   [junit4]   2> 1663394 DEBUG (simCloudManagerPool-11488-thread-6) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=2
   [junit4]   2> 1663395 DEBUG (simCloudManagerPool-11488-thread-6) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 2
   [junit4]   2> 1663405 DEBUG (simCloudManagerPool-11488-thread-7) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=3
   [junit4]   2> 1663406 DEBUG (simCloudManagerPool-11488-thread-7) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 3
   [junit4]   2> 1663416 DEBUG (simCloudManagerPool-11488-thread-8) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=4
   [junit4]   2> 1663416 DEBUG (simCloudManagerPool-11488-thread-8) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 4
   [junit4]   2> 1663427 DEBUG (simCloudManagerPool-11488-thread-9) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=5
   [junit4]   2> 1663427 DEBUG (simCloudManagerPool-11488-thread-9) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 5
   [junit4]   2> 1663427 DEBUG (simCloudManagerPool-11488-thread-6) [    ] o.a.s.c.a.s.SimClusterStateProvider -- finished createCollection .system, currentVersion=6
   [junit4]   2> 1663427 INFO  (simCloudManagerPool-11488-thread-6) [    ] o.a.s.c.CloudTestUtils -- shards without leaders: [shard1]
   [junit4]   2> 1663427 DEBUG (simCloudManagerPool-11488-thread-10) [    ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for .system / shard1 (currentVersion=6): {"core_node1":{
   [junit4]   2>     "core":".system_shard1_replica_n1",
   [junit4]   2>     "shard":"shard1",
   [junit4]   2>     "collection":".system",
   [junit4]   2>     "node_name":"127.0.0.1:10277_solr",
   [junit4]   2>     "type":"NRT",
   [junit4]   2>     "leader":"true",
   [junit4]   2>     "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>     "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>     "INDEX.sizeInBytes":10240,
   [junit4]   2>     "state":"active",
   [junit4]   2>     "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>     "SEARCHER.searcher.numDocs":0}}
   [junit4]   2> 1663429 DEBUG (simCloudManagerPool-11488-thread-6) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=6
   [junit4]   2> 1663429 DEBUG (simCloudManagerPool-11488-thread-6) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 6
   [junit4]   2> 1663430 DEBUG (ScheduledTrigger-11489-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Removing new node from tracking: 127.0.0.1:10279_solr
   [junit4]   2> 1663491 DEBUG (TEST-TestSimTriggerIntegration.testListeners-seed#[77FFAC40FA650924]) [    ] o.a.s.c.a.s.SimClusterStateProvider -- creating marker: /autoscaling/nodeAdded/127.0.0.1:10280_solr
   [junit4]   2> 1663531 DEBUG (ScheduledTrigger-11489-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 and last live nodes: 3
   [junit4]   2> 1663531 DEBUG (ScheduledTrigger-11489-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1663531 DEBUG (ScheduledTrigger-11489-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
   [junit4]   2> 1663531 DEBUG (ScheduledTrigger-11489-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10280_solr at time 615692193463194
   [junit4]   2> 1663551 DEBUG (ScheduledTrigger-11489-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1663551 DEBUG (ScheduledTrigger-11489-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 and last live nodes: 4
   [junit4]   2> 1663551 DEBUG (ScheduledTrigger-11489-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
   [junit4]   2> 1663571 DEBUG (ScheduledTrigger-11489-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1663571 DEBUG (ScheduledTrigger-11489-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 and last live nodes: 4
   [junit4]   2> 1663571 DEBUG (ScheduledTrigger-11489-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
   [junit4]   2> 1663592 DEBUG (ScheduledTrigger-11489-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1663592 DEBUG (ScheduledTrigger-11489-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 and last live nodes: 4
   [junit4]   2> 1663592 DEBUG (ScheduledTrigger-11489-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
   [junit4]   2> 1663592 DEBUG (ScheduledTrigger-11489-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger firing registered processor for nodes: [127.0.0.1:10280_solr] added at times [615692193463194], now=615695221289294
   [junit4]   2> 1663592 DEBUG (ScheduledTrigger-11489-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Removing new node from tracking: 127.0.0.1:10280_solr
   [junit4]   2> 1663593 WARN  (AutoscalingActionExecutor-11490-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Exception executing actions
   [junit4]   2> org.apache.solr.cloud.autoscaling.TriggerActionException: Error processing action for trigger event: {
   [junit4]   2>   "id":"22ff8027b7b9aTdy6qptrtyaa1mq11kfykwnh9e",
   [junit4]   2>   "source":"node_added_trigger",
   [junit4]   2>   "eventTime":615692193463194,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[615692193463194],
   [junit4]   2>     "preferredOperation":"movereplica",
   [junit4]   2>     "_enqueue_time_":615695229988644,
   [junit4]   2>     "nodeNames":["127.0.0.1:10280_solr"]}}
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$add$3(ScheduledTriggers.java:315) [java/:?]
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:514) [?:?]
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?]
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) [java/:?]
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1167) [?:?]
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641) [?:?]
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:844) [?:?]
   [junit4]   2> Caused by: java.lang.RuntimeException: failure
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration$TestDummyAction.process(TestSimTriggerIntegration.java:942) ~[test/:?]
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$add$3(ScheduledTriggers.java:312) ~[java/:?]
   [junit4]   2> 	... 6 more
   [junit4]   2> 1663653 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 1663653 DEBUG (TEST-TestSimTriggerIntegration.testListeners-seed#[77FFAC40FA650924]) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 1663658 DEBUG (TEST-TestSimTriggerIntegration.testNodeLostTriggerRestoreState-seed#[77FFAC40FA650924]) [    ] o.a.s.c.a.s.SimClusterStateProvider --- new Overseer leader: 127.0.0.1:10281_solr
   [junit4]   2> 1663658 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
   [junit4]   2> 1663658 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
   [junit4]   2> 1663658 DEBUG (TEST-TestSimTriggerIntegration.testNodeLostTriggerRestoreState-seed#[77FFAC40FA650924]) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=0
   [junit4]   2> 1663658 DEBUG (TEST-TestSimTriggerIntegration.testNodeLostTriggerRestoreState-seed#[77FFAC40FA650924]) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 0
   [junit4]   2> 1663658 INFO  (TEST-TestSimTriggerIntegration.testNodeLostTriggerRestoreState-seed#[77FFAC40FA650924]) [    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 1663658 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 0
   [junit4]   2> 1663658 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
   [junit4]   2> 1663658 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10281_solr, 127.0.0.1:10282_solr]
   [junit4]   2> 1663658 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
   [junit4]   2> 1663658 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
   [junit4]   2> 1663658 DEBUG (ScheduledTrigger-11495-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
   [junit4]   2> 1663659 DEBUG (simCloudManagerPool-11494-thread-2) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1663659 DEBUG (simCloudManagerPool-11494-thread-2) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2
   [junit4]   2> 1663659 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
   [junit4]   2> 1663659 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
   [junit4]   2> 1663660 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
   [junit4]   2> 1663660 DEBUG (simCloudManagerPool-11494-thread-3) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1663660 DEBUG (simCloudManagerPool-11494-thread-3) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 3
   [junit4]   2> 1663660 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
   [junit4]   2> 1663660 DEBUG (TEST-TestSimTriggerIntegration.testNodeLostTriggerRestoreState-seed#[77FFAC40FA650924]) [    ] o.a.s.c.a.s.SimClusterStateProvider  -- created marker: /autoscaling/nodeLost/127.0.0.1:10283_solr
   [junit4]   2> 1663660 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.TestSimTriggerIntegration TestTriggerAction init
   [junit4]   2> 1663660 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger node_lost_restore_trigger - Initial livenodes: [127.0.0.1:10281_solr, 127.0.0.1:10282_solr]
   [junit4]   2> 1663660 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger Adding lost node from marker path: 127.0.0.1:10283_solr
   [junit4]   2> 1663661 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
   [junit4]   2> 1663661 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
   [junit4]   2> 1663661 DEBUG (ScheduledTrigger-11495-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_restore_trigger with currently live nodes: 2 and last live nodes: 2
   [junit4]   2> 1663679 DEBUG (ScheduledTrigger-11495-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
   [junit4]   2> 1663681 DEBUG (ScheduledTrigger-11495-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_restore_trigger with currently live nodes: 2 and last live nodes: 2
   [junit4]   2> 1663691 DEBUG (simCloudManagerPool-11494-thread-4) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1663691 DEBUG (simCloudManagerPool-11494-thread-4) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 4
   [junit4]   2> 1663692 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 4
   [junit4]   2> 1663692 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.TestSimTriggerIntegration TestTriggerAction init
   [junit4]   2> 1663692 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger node_lost_restore_trigger - Initial livenodes: [127.0.0.1:10281_solr, 127.0.0.1:10282_solr]
   [junit4]   2> 1663692 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
   [junit4]   2> 1663692 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion 4
   [junit4]   2> 1663692 DEBUG (ScheduledTrigger-11495-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_restore_trigger with currently live nodes: 2 and last live nodes: 2
   [junit4]   2> 1663692 DEBUG (ScheduledTrigger-11495-thread-1) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:10283_solr]
   [junit4]   2> 1663693 DEBUG (simCloudManagerPool-11494-thread-5) [    ] o.a.s.c.a.s.SimClusterStateProvider -- simCreateCollection .system, currentVersion=1
   [junit4]   2> 1663693 DEBUG (simCloudManagerPool-11494-thread-5) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=1
   [junit4]   2> 1663693 DEBUG (simCloudManagerPool-11494-thread-5) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 1
   [junit4]   2> 1663703 DEBUG (simCloudManagerPool-11494-thread-5) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=2
   [junit4]   2> 1663703 DEBUG (simCloudManagerPool-11494-thread-5) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 2
   [junit4]   2> 1663714 DEBUG (simCloudManagerPool-11494-thread-6) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=3
   [junit4]   2> 1663714 DEBUG (simCloudManagerPool-11494-thread-6) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 3
   [junit4]   2> 1663724 DEBUG (simCloudManagerPool-11494-thread-7) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=4
   [junit4]   2> 1663724 DEBUG (simCloudManagerPool-11494-thread-7) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 4
   [junit4]   2> 1663724 DEBUG (simCloudManagerPool-11494-thread-5) [    ] o.a.s.c.a.s.SimClusterStateProvider -- finished createCollection .system, currentVersion=5
   [junit4]   2> 1663724 INFO  (simCloudManagerPool-11494-thread-5) [    ] o.a.s.c.CloudTestUtils -- shards without leaders: [shard1]
   [junit4]   2> 1663724 DEBUG (simCloudManagerPool-11494-thread-8) [    ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for .system / shard1 (currentVersion=5): {"core_node1":{
   [junit4]   2>     "core":".system_shard1_replica_n1",
   [junit4]   2>     "shard":"shard1",
   [junit4]   2>     "collection":".system",
   [junit4]   2>     "node_name":"127.0.0.1:10282_solr",
   [junit4]   2>     "type":"NRT",
   [junit4]   2>     "leader":"true",
   [junit4]   2>     "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>     "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>     "INDEX.sizeInBytes":10240,
   [junit4]   2>     "state":"active",
   [junit4]   2>     "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>     "SEARCHER.searcher.numDocs":0}}
   [junit4]   2> 1663726 DEBUG (simCloudManagerPool-11494-thread-5) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=5
   [junit4]   2> 1663727 DEBUG (simCloudManagerPool-11494-thread-5) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 5
   [junit4]   2> 1663727 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 1663727 WARN  (AutoscalingActionExecutor-11496-thread-1) [    ] o.a.s.c.a.SystemLogListener Exception sending event to collection .system
   [junit4]   2> java.io.IOException: java.util.concurrent.ExecutionException: java.io.IOException: already closed
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.request(SimCloudManager.java:654) ~[test/:?]
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.SystemLogListener.onEvent(SystemLogListener.java:118) [java/:?]
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.ScheduledTriggers$TriggerListeners.fireListeners(ScheduledTriggers.java:807) [java/:?]
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.ScheduledTriggers$TriggerListeners.fireListeners(ScheduledTriggers.java:774) [java/:?]
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$add$3(ScheduledTriggers.java:325) [java/:?]
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:514) [?:?]
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?]
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) [java/:?]
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1167) [?:?]
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641) [?:?]
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:844) [?:?]
   [junit4]   2> Caused by: java.util.concurrent.ExecutionException: java.io.IOException: already closed
   [junit4]   2> 	at java.util.concurrent.FutureTask.report(FutureTask.java:122) ~[?:?]
   [junit4]   2> 	at java.util.concurrent.FutureTask.get(FutureTask.java:191) ~[?:?]
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.request(SimCloudManager.java:652) ~[test/:?]
   [junit4]   2> 	... 10 more
   [junit4]   2> Caused by: java.io.IOException: already closed
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.sim.SimClusterStateProvider.ensureNotClosed(SimClusterStateProvider.java:2208) ~[test/:?]
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.sim.SimClusterStateProvider.simUpdate(SimClusterStateProvider.java:1425) ~[test/:?]
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.simHandleSolrRequest(SimCloudManager.java:741) ~[test/:?]
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.lambda$request$15(SimCloudManager.java:651) ~[test/:?]
   [junit4]   2> 	... 5 more
   [junit4]   2> 1663728 DEBUG (TEST-TestSimTriggerIntegration.testNodeLostTriggerRestoreState-seed#[77FFAC40FA650924]) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4] IGNOR/A 0.00s J2 | TestSimTriggerIntegration.testSearchRate
   [junit4]    > Assumption #1: 'awaitsfix' test group is disabled (@AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028"))
   [junit4]   2> 1663732 DEBUG (TEST-TestSimTriggerIntegration.testTriggerThrottling-seed#[77FFAC40FA650924]) [    ] o.a.s.c.a.s.SimClusterStateProvider --- new Overseer leader: 127.0.0.1:10284_solr
   [junit4]   2> 1663732 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
   [junit4]   2> 1663732 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
   [junit4]   2> 1663732 DEBUG (TEST-TestSimTriggerIntegration.testTriggerThrottling-seed#[77FFAC40FA650924]) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=0
   [junit4]   2> 1663732 DEBUG (TEST-TestSimTriggerIntegration.testTriggerThrottling-seed#[77FFAC40FA650924]) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 0
   [junit4]   2> 1663732 INFO  (TEST-TestSimTriggerIntegration.testTriggerThrottling-seed#[77FFAC40FA650924]) [    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 1663732 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 0
   [junit4]   2> 1663732 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
   [junit4]   2> 1663732 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10284_solr]
   [junit4]   2> 1663732 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
   [junit4]   2> 1663732 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
   [junit4]   2> 1663732 DEBUG (ScheduledTrigger-11501-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 1
   [junit4]   2> 1663733 DEBUG (simCloudManagerPool-11500-thread-2) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1663733 DEBUG (simCloudManagerPool-11500-thread-2) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2
   [junit4]   2> 1663733 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
   [junit4]   2> 1663733 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
   [junit4]   2> 1663733 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
   [junit4]   2> 1663734 DEBUG (simCloudManagerPool-11500-thread-3) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1663734 DEBUG (simCloudManagerPool-11500-thread-3) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 3
   [junit4]   2> 1663734 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
   [junit4]   2> 1663734 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.TestSimTriggerIntegration TestTriggerAction init
   [junit4]   2> 1663734 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 - Initial livenodes: [127.0.0.1:10284_solr, 127.0.0.1:10285_solr]
   [junit4]   2> 1663734 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 instantiated with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration$ThrottlingTesterAction}], enabled=true}
   [junit4]   2> 1663734 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
   [junit4]   2> 1663734 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
   [junit4]   2> 1663734 DEBUG (ScheduledTrigger-11501-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 1663734 DEBUG (ScheduledTrigger-11501-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
   [junit4]   2> 1663734 DEBUG (simCloudManagerPool-11500-thread-4) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1663734 DEBUG (simCloudManagerPool-11500-thread-4) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 4
   [junit4]   2> 1663735 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 4
   [junit4]   2> 1663735 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.TestSimTriggerIntegration TestTriggerAction init
   [junit4]   2> 1663735 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger2 - Initial livenodes: [127.0.0.1:10284_solr, 127.0.0.1:10285_solr]
   [junit4]   2> 1663735 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger2 instantiated with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration$ThrottlingTesterAction}], enabled=true}
   [junit4]   2> 1663735 DEBUG (TEST-TestSimTriggerIntegration.testTriggerThrottling-seed#[77FFAC40FA650924]) [    ] o.a.s.c.a.s.SimClusterStateProvider -- creating marker: /autoscaling/nodeAdded/127.0.0.1:10286_solr
   [junit4]   2> 1663735 DEBUG (ScheduledTrigger-11501-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
   [junit4]   2> 1663735 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
   [junit4]   2> 1663735 DEBUG (ScheduledTrigger-11501-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1663735 DEBUG (ScheduledTrigger-11501-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10286_solr at time 615702377306144
   [junit4]   2> 1663735 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion 4
   [junit4]   2> 1663735 DEBUG (ScheduledTrigger-11501-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger2 firing registered processor for nodes: [127.0.0.1:10286_solr] added at times [615702377306144], now=615702378084744
   [junit4]   2> 1663735 DEBUG (simCloudManagerPool-11500-thread-5) [    ] o.a.s.c.a.s.SimClusterStateProvider -- simCreateCollection .system, currentVersion=1
   [junit4]   2> 1663735 DEBUG (simCloudManagerPool-11500-thread-5) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=1
   [junit4]   2> 1663735 DEBUG (simCloudManagerPool-11500-thread-5) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 1
   [junit4]   2> 1663745 DEBUG (simCloudManagerPool-11500-thread-5) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=2
   [junit4]   2> 1663746 DEBUG (simCloudManagerPool-11500-thread-5) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 2
   [junit4]   2> 1663756 DEBUG (simCloudManagerPool-11500-thread-6) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=3
   [junit4]   2> 1663756 DEBUG (simCloudManagerPool-11500-thread-6) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 3
   [junit4]   2> 1663767 DEBUG (simCloudManagerPool-11500-thread-7) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=4
   [junit4]   2> 1663767 DEBUG (simCloudManagerPool-11500-thread-7) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 4
   [junit4]   2> 1663767 DEBUG (simCloudManagerPool-11500-thread-10) [    ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for .system / shard1 (currentVersion=5): {"core_node1":{
   [junit4]   2>     "core":".system_shard1_replica_n1",
   [junit4]   2>     "shard":"shard1",
   [junit4]   2>     "collection":".system",
   [junit4]   2>     "node_name":"127.0.0.1:10285_solr",
   [junit4]   2>     "type":"NRT",
   [junit4]   2>     "leader":"true",
   [junit4]   2>     "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>     "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>     "INDEX.sizeInBytes":10240,
   [junit4]   2>     "state":"active",
   [junit4]   2>     "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>     "SEARCHER.searcher.numDocs":0}}
   [junit4]   2> 1663777 DEBUG (simCloudManagerPool-11500-thread-8) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=5
   [junit4]   2> 1663778 DEBUG (simCloudManagerPool-11500-thread-8) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 5
   [junit4]   2> 1663780 DEBUG (simCloudManagerPool-11500-thread-5) [    ] o.a.s.c.a.s.SimClusterStateProvider -- finished createCollection .system, currentVersion=6
   [junit4]   2> 1663780 DEBUG (simCloudManagerPool-11500-thread-5) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=6
   [junit4]   2> 1663780 DEBUG (simCloudManagerPool-11500-thread-5) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 6
   [junit4]   2> 1663781 DEBUG (ScheduledTrigger-11501-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Removing new node from tracking: 127.0.0.1:10286_solr
   [junit4]   2> 1663781 INFO  (AutoscalingActionExecutor-11502-thread-1) [    ] o.a.s.c.a.s.TestSimTriggerIntegration action executed from node_added_trigger2
   [junit4]   2> 1663882 DEBUG (ScheduledTrigger-11501-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 2
   [junit4]   2> 1663882 DEBUG (ScheduledTrigger-11501-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 1663882 DEBUG (ScheduledTrigger-11501-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
   [junit4]   2> 1663882 DEBUG (ScheduledTrigger-11501-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1663882 DEBUG (ScheduledTrigger-11501-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1663882 DEBUG (ScheduledTrigger-11501-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10286_solr at time 615709758288144
   [junit4]   2> 1663882 DEBUG (ScheduledTrigger-11501-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 firing registered processor for nodes: [127.0.0.1:10286_solr] added at times [615709758288144], now=615709760339544
   [junit4]   2> 1663883 DEBUG (ScheduledTrigger-11501-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Removing new node from tracking: 127.0.0.1:10286_solr
   [junit4]   2> 1663883 INFO  (AutoscalingActionExecutor-11502-thread-1) [    ] o.a.s.c.a.s.TestSimTriggerIntegration last action at 615704698207544 time = 615709794431394
   [junit4]   2> 1663883 INFO  (AutoscalingActionExecutor-11502-thread-1) [    ] o.a.s.c.a.s.TestSimTriggerIntegration action executed from node_added_trigger1
   [junit4]   2> 1663884 DEBUG (simCloudManagerPool-11500-thread-13) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1663884 DEBUG (simCloudManagerPool-11500-thread-13) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 5
   [junit4]   2> 1663884 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 5
   [junit4]   2> 1663884 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.TestSimTriggerIntegration TestTriggerAction init
   [junit4]   2> 1663884 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger node_lost_trigger1 - Initial livenodes: [127.0.0.1:10284_solr, 127.0.0.1:10286_solr, 127.0.0.1:10285_solr]
   [junit4]   2> 1663884 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
   [junit4]   2> 1663884 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 5, lastZnodeVersion 5
   [junit4]   2> 1663884 DEBUG (ScheduledTrigger-11501-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with currently live nodes: 3 and last live nodes: 3
   [junit4]   2> 1663885 DEBUG (simCloudManagerPool-11500-thread-15) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1663885 DEBUG (simCloudManagerPool-11500-thread-15) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 6
   [junit4]   2> 1663885 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 6
   [junit4]   2> 1663885 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.TestSimTriggerIntegration TestTriggerAction init
   [junit4]   2> 1663885 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger node_lost_trigger2 - Initial livenodes: [127.0.0.1:10284_solr, 127.0.0.1:10286_solr, 127.0.0.1:10285_solr]
   [junit4]   2> 1663885 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
   [junit4]   2> 1663885 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion 6
   [junit4]   2> 1663885 DEBUG (TEST-TestSimTriggerIntegration.testTriggerThrottling-seed#[77FFAC40FA650924]) [    ] o.a.s.c.a.s.SimClusterStateProvider  -- created marker: /autoscaling/nodeLost/127.0.0.1:10286_solr
   [junit4]   2> 1663885 DEBUG (ScheduledTrigger-11501-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger2 with currently live nodes: 2 and last live nodes: 3
   [junit4]   2> 1663885 DEBUG (TEST-TestSimTriggerIntegration.testTriggerThrottling-seed#[77FFAC40FA650924]) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=7
   [junit4]   2> 1663885 DEBUG (ScheduledTrigger-11501-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10286_solr
   [junit4]   2> 1663885 DEBUG (ScheduledTrigger-11501-thread-1) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:10286_solr]
   [junit4]   2> 1663885 DEBUG (TEST-TestSimTriggerIntegration.testTriggerThrottling-seed#[77FFAC40FA650924]) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 7
   [junit4]   2> 1663886 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 1663886 WARN  (ScheduledTrigger-11501-thread-1) [    ] o.a.s.c.a.SystemLogListener Exception sending event to collection .system
   [junit4]   2> java.io.IOException: java.util.concurrent.ExecutionException: java.io.IOException: already closed
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.request(SimCloudManager.java:654) ~[test/:?]
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.SystemLogListener.onEvent(SystemLogListener.java:118) [java/:?]
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.ScheduledTriggers$TriggerListeners.fireListeners(ScheduledTriggers.java:807) [java/:?]
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.ScheduledTriggers$TriggerListeners.fireListeners(ScheduledTriggers.java:774) [java/:?]
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$add$4(ScheduledTriggers.java:285) [java/:?]
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.NodeLostTrigger.run(NodeLostTrigger.java:185) [java/:?]
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.ScheduledTriggers$TriggerWrapper.run(ScheduledTriggers.java:604) [java/:?]
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:514) [?:?]
   [junit4]   2> 	at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) [?:?]
   [junit4]   2> 	at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:300) [?:?]
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1167) [?:?]
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641) [?:?]
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:844) [?:?]
   [junit4]   2> Caused by: java.util.concurrent.ExecutionException: java.io.IOException: already closed
   [junit4]   2> 	at java.util.concurrent.FutureTask.report(FutureTask.java:122) ~[?:?]
   [junit4]   2> 	at java.util.concurrent.FutureTask.get(FutureTask.java:191) ~[?:?]
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.request(SimCloudManager.java:652) ~[test/:?]
   [junit4]   2> 	... 12 more
   [junit4]   2> Caused by: java.io.IOException: already closed
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.sim.SimClusterStateProvider.ensureNotClosed(SimClusterStateProvider.java:2208) ~[test/:?]
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.sim.SimClusterStateProvider.simUpdate(SimClusterStateProvider.java:1425) ~[test/:?]
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.simHandleSolrRequest(SimCloudManager.java:741) ~[test/:?]
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.lambda$request$15(SimCloudManager.java:651) ~[test/:?]
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) ~[java/:?]
   [junit4]   2> 	... 3 more
   [junit4]   2> 1663887 WARN  (ScheduledTrigger-11501-thread-1) [    ] o.a.s.c.a.SystemLogListener Exception sending event to collection .system
   [junit4]   2> java.io.IOException: java.lang.InterruptedException
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.request(SimCloudManager.java:654) ~[test/:?]
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.SystemLogListener.onEvent(SystemLogListener.java:118) [java/:?]
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.ScheduledTriggers$TriggerListeners.fireListeners(ScheduledTriggers.java:807) [java/:?]
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.ScheduledTriggers$TriggerListeners.fireListeners(ScheduledTriggers.java:778) [java/:?]
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$add$4(ScheduledTriggers.java:290) [java/:?]
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.NodeLostTrigger.run(NodeLostTrigger.java:185) [java/:?]
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.ScheduledTriggers$TriggerWrapper.run(ScheduledTriggers.java:604) [java/:?]
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:514) [?:?]
   [junit4]   2> 	at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) [?:?]
   [junit4]   2> 	at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:300) [?:?]
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1167) [?:?]
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641) [?:?]
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:844) [?:?]
   [junit4]   2> Caused by: java.lang.InterruptedException
   [junit4]   2> 	at java.util.concurrent.FutureTask.awaitDone(FutureTask.java:418) ~[?:?]
   [junit4]   2> 	at java.util.concurrent.FutureTask.get(FutureTask.java:190) ~[?:?]
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.request(SimCloudManager.java:652) ~[test/:?]
   [junit4]   2> 	... 12 more
   [junit4]   2> 1663887 WARN  (ScheduledTrigger-11501-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Ignoring autoscaling event {
   [junit4]   2>   "id":"22ffc22085646Tdy6qptrtyaa1mq11kfykwnh9s",
   [junit4]   2>   "source":"node_lost_trigger2",
   [junit4]   2>   "eventTime":615709902657094,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[615709902657094],
   [junit4]   2>     "preferredOperation":"movereplica",
   [junit4]   2>     "_enqueue_time_":615709908332094,
   [junit4]   2>     "nodeNames":["127.0.0.1:10286_solr"]}} from trigger org.apache.solr.cloud.autoscaling.NodeLostTrigger@161b5b6d because the executor has already been closed
   [junit4]   2> 1663887 DEBUG (ScheduledTrigger-11501-thread-1) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger processor for lost nodes: [127.0.0.1:10286_solr] is not ready, will try later
   [junit4]   2> 1663887 DEBUG (TEST-TestSimTriggerIntegration.testTriggerThrottling-seed#[77FFAC40FA650924]) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4] IGNOR/A 0.00s J2 | TestSimTriggerIntegration.testEventQueue
   [junit4]    > Assumption #1: 'awaitsfix' test group is disabled (@AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028"))
   [junit4]   2> 1663893 DEBUG (TEST-TestSimTriggerIntegration.testEventFromRestoredState-seed#[77FFAC40FA650924]) [    ] o.a.s.c.a.s.SimClusterStateProvider --- new Overseer leader: 127.0.0.1:10287_solr
   [junit4]   2> 1663893 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
   [junit4]   2> 1663893 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
   [junit4]   2> 1663893 DEBUG (TEST-TestSimTriggerIntegration.testEventFromRestoredState-seed#[77FFAC40FA650924]) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=0
   [junit4]   2> 1663893 DEBUG (TEST-TestSimTriggerIntegration.testEventFromRestoredState-seed#[77FFAC40FA650924]) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 0
   [junit4]   2> 1663893 INFO  (TEST-TestSimTriggerIntegration.testEventFromRestoredState-seed#[77FFAC40FA650924]) [    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 1663893 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 0
   [junit4]   2> 1663893 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
   [junit4]   2> 1663894 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10287_solr, 127.0.0.1:10288_solr]
   [junit4]   2> 1663897 DEBUG (ScheduledTrigger-11507-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
   [junit4]   2> 1663901 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
   [junit4]   2> 1663901 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
   [junit4]   2> 1663905 DEBUG (simCloudManagerPool-11506-thread-2) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1663905 DEBUG (simCloudManagerPool-11506-thread-2) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2
   [junit4]   2> 1663906 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
   [junit4]   2> 1663909 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
   [junit4]   2> 1663909 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
   [junit4]   2> 1663910 DEBUG (simCloudManagerPool-11506-thread-3) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1663910 DEBUG (simCloudManagerPool-11506-thread-3) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 3
   [junit4]   2> 1663910 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
   [junit4]   2> 1663910 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.TestSimTriggerIntegration TestTriggerAction init
   [junit4]   2> 1663910 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger - Initial livenodes: [127.0.0.1:10287_solr, 127.0.0.1:10288_solr]
   [junit4]   2> 1663910 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated with properties: {event=nodeAdded, waitFor=10, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration$TestTriggerAction}], enabled=true}
   [junit4]   2> 1663910 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
   [junit4]   2> 1663910 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
   [junit4]   2> 1663910 DEBUG (TEST-TestSimTriggerIntegration.testEventFromRestoredState-seed#[77FFAC40FA650924]) [    ] o.a.s.c.a.s.SimClusterStateProvider -- creating marker: /autoscaling/nodeAdded/127.0.0.1:10289_solr
   [junit4]   2> 1663910 DEBUG (ScheduledTrigger-11507-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1663910 DEBUG (ScheduledTrigger-11507-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1663910 DEBUG (ScheduledTrigger-11507-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10289_solr at time 615711149857544
   [junit4]   2> 1663917 DEBUG (ScheduledTrigger-11507-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 2
   [junit4]   2> 1663931 DEBUG (ScheduledTrigger-11507-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1663931 DEBUG (ScheduledTrigger-11507-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1663937 DEBUG (ScheduledTrigger-11507-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
   [junit4]   2> 1663951 DEBUG (ScheduledTrigger-11507-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1663951 DEBUG (ScheduledTrigger-11507-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1663958 DEBUG (ScheduledTrigger-11507-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
   [junit4]   2> 1663971 DEBUG (ScheduledTrigger-11507-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1663971 DEBUG (ScheduledTrigger-11507-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1663978 DEBUG (ScheduledTrigger-11507-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
   [junit4]   2> 1663991 DEBUG (ScheduledTrigger-11507-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1663991 DEBUG (ScheduledTrigger-11507-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1663998 DEBUG (ScheduledTrigger-11507-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
   [junit4]   2> 1664011 DEBUG (ScheduledTrigger-11507-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1664011 DEBUG (ScheduledTrigger-11507-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1664018 DEBUG (ScheduledTrigger-11507-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
   [junit4]   2> 1664031 DEBUG (ScheduledTrigger-11507-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1664032 DEBUG (ScheduledTrigger-11507-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1664038 DEBUG (ScheduledTrigger-11507-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
   [junit4]   2> 1664052 DEBUG (ScheduledTrigger-11507-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1664052 DEBUG (ScheduledTrigger-11507-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1664058 DEBUG (ScheduledTrigger-11507-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
   [junit4]   2> 1664072 DEBUG (ScheduledTrigger-11507-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1664072 DEBUG (ScheduledTrigger-11507-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1664079 DEBUG (ScheduledTrigger-11507-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
   [junit4]   2> 1664092 DEBUG (ScheduledTrigger-11507-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1664092 DEBUG (ScheduledTrigger-11507-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1664099 DEBUG (ScheduledTrigger-11507-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
   [junit4]   2> 1664112 DEBUG (ScheduledTrigger-11507-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1664112 DEBUG (ScheduledTrigger-11507-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1664112 DEBUG (ScheduledTrigger-11507-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger firing registered processor for nodes: [127.0.0.1:10289_solr] added at times [615711149857544], now=615721255960194
   [junit4]   2> 1664113 DEBUG (simCloudManagerPool-11506-thread-4) [    ] o.a.s.c.a.s.SimClusterStateProvider -- simCreateCollection .system, currentVersion=1
   [junit4]   2> 1664113 DEBUG (simCloudManagerPool-11506-thread-4) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=1
   [junit4]   2> 1664113 DEBUG (simCloudManagerPool-11506-thread-4) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 1
   [junit4]   2> 1664124 DEBUG (simCloudManagerPool-11506-thread-4) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=2
   [junit4]   2> 1664124 DEBUG (simCloudManagerPool-11506-thread-4) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 2
   [junit4]   2> 1664134 DEBUG (simCloudManagerPool-11506-thread-5) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=3
   [junit4]   2> 1664135 DEBUG (simCloudManagerPool-11506-thread-5) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 3
   [junit4]   2> 1664145 DEBUG (simCloudManagerPool-11506-thread-6) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=4
   [junit4]   2> 1664145 DEBUG (simCloudManagerPool-11506-thread-6) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 4
   [junit4]   2> 1664156 DEBUG (simCloudManagerPool-11506-thread-7) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=5
   [junit4]   2> 1664156 DEBUG (simCloudManagerPool-11506-thread-7) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 5
   [junit4]   2> 1664156 DEBUG (simCloudManagerPool-11506-thread-4) [    ] o.a.s.c.a.s.SimClusterStateProvider -- finished createCollection .system, currentVersion=6
   [junit4]   2> 1664156 INFO  (simCloudManagerPool-11506-thread-4) [    ] o.a.s.c.CloudTestUtils -- shards without leaders: [shard1]
   [junit4]   2> 1664156 DEBUG (simCloudManagerPool-11506-thread-8) [    ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for .system / shard1 (currentVersion=6): {"core_node1":{
   [junit4]   2>     "core":".system_shard1_replica_n1",
   [junit4]   2>     "shard":"shard1",
   [junit4]   2>     "collection":".system",
   [junit4]   2>     "node_name":"127.0.0.1:10289_solr",
   [junit4]   2>     "type":"NRT",
   [junit4]   2>     "leader":"true",
   [junit4]   2>     "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>     "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>     "INDEX.sizeInBytes":10240,
   [junit4]   2>     "state":"active",
   [junit4]   2>     "INDEX.sizeInGB":9.5367431640625E-6,
   [junit4]   2>     "SEARCHER.searcher.numDocs":0}}
   [junit4]   2> 1664158 DEBUG (simCloudManagerPool-11506-thread-4) [    ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=6
   [junit4]   2> 1664158 DEBUG (simCloudManagerPool-11506-thread-4) [    ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 6
   [junit4]   2> 1664159 DEBUG (ScheduledTrigger-11507-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Removing new node from tracking: 127.0.0.1:10289_solr
   [junit4]   2> 1664159 DEBUG (TEST-TestSimTriggerIntegration.testEventFromRestoredState-seed#[77FFAC40FA650924]) [    ] o.a.s.c.a.s.SimClusterStateProvider -- creating marker: /autoscaling/nodeAdded/127.0.0.1:10290_solr
   [junit4]   2> 1664259 DEBUG (ScheduledTrigger-11507-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 and last live nodes: 3
   [junit4]   2> 1664259 DEBUG (ScheduledTrigger-11507-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1664259 DEBUG (ScheduledTrigger-11507-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
   [junit4]   2> 1664259 DEBUG (ScheduledTrigger-11507-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10290_solr at time 615728610108694
   [junit4]   2> 1664280 DEBUG (ScheduledTrigger-11507-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1664280 DEBUG (ScheduledTrigger-11507-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 and last live nodes: 4
   [junit4]   2> 1664280 DEBUG (ScheduledTrigger-11507-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
   [junit4]   2> 1664300 DEBUG (ScheduledTrigger-11507-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 and last live nodes: 4
   [junit4]   2> 1664300 DEBUG (ScheduledTrigger-11507-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1664300 DEBUG (ScheduledTrigger-11507-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
   [junit4]   2> 1664320 DEBUG (ScheduledTrigger-11507-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 and last live nodes: 4
   [junit4]   2> 1664320 DEBUG (ScheduledTrigger-11507-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1664320 DEBUG (ScheduledTrigger-11507-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
   [junit4]   2> 1664340 DEBUG (ScheduledTrigger-11507-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 and last live nodes: 4
   [junit4]   2> 1664340 DEBUG (ScheduledTrigger-11507-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1664340 DEBUG (ScheduledTrigger-11507-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
   [junit4]   2> 1664359 INFO  (TEST-TestSimTriggerIntegration.testEventFromRestoredState-seed#[77FFAC40FA650924]) [    ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
   [junit4]   2> 1664359 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 1664360 DEBUG (TEST-TestSimTr

[...truncated too long message...]

gger
   [junit4]   2> 10808 DEBUG (ScheduledTrigger-110-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 10828 DEBUG (ScheduledTrigger-110-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 3 and last live nodes: 3
   [junit4]   2> 10828 DEBUG (ScheduledTrigger-110-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 10828 DEBUG (ScheduledTrigger-110-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
   [junit4]   2> 10828 DEBUG (ScheduledTrigger-110-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 10848 DEBUG (ScheduledTrigger-110-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 3 and last live nodes: 3
   [junit4]   2> 10848 DEBUG (ScheduledTrigger-110-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
   [junit4]   2> 10848 DEBUG (ScheduledTrigger-110-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 10848 DEBUG (ScheduledTrigger-110-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 10869 DEBUG (ScheduledTrigger-110-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 3 and last live nodes: 3
   [junit4]   2> 10869 DEBUG (ScheduledTrigger-110-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 10869 DEBUG (ScheduledTrigger-110-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
   [junit4]   2> 10869 DEBUG (ScheduledTrigger-110-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 10889 DEBUG (ScheduledTrigger-110-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 3 and last live nodes: 3
   [junit4]   2> 10889 DEBUG (ScheduledTrigger-110-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 10889 DEBUG (ScheduledTrigger-110-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
   [junit4]   2> 10889 DEBUG (ScheduledTrigger-110-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 10909 DEBUG (ScheduledTrigger-110-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
   [junit4]   2> 10909 DEBUG (ScheduledTrigger-110-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 10909 DEBUG (ScheduledTrigger-110-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 3 and last live nodes: 3
   [junit4]   2> 10909 DEBUG (ScheduledTrigger-110-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 10929 DEBUG (ScheduledTrigger-110-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
   [junit4]   2> 10929 DEBUG (ScheduledTrigger-110-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 3 and last live nodes: 3
   [junit4]   2> 10929 DEBUG (ScheduledTrigger-110-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 10929 DEBUG (ScheduledTrigger-110-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 10941 DEBUG (TEST-TestSimTriggerIntegration.testNodeMarkersRegistration-seed#[77FFAC40FA650924]) [    ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
   [junit4]   2> 10941 DEBUG (TEST-TestSimTriggerIntegration.testNodeMarkersRegistration-seed#[77FFAC40FA650924]) [    ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now
   [junit4]   2> 10941 DEBUG (TEST-TestSimTriggerIntegration.testNodeMarkersRegistration-seed#[77FFAC40FA650924]) [    ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
   [junit4]   2> 10941 DEBUG (TEST-TestSimTriggerIntegration.testNodeMarkersRegistration-seed#[77FFAC40FA650924]) [    ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor
   [junit4]   2> 10941 DEBUG (TEST-TestSimTriggerIntegration.testNodeMarkersRegistration-seed#[77FFAC40FA650924]) [    ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
   [junit4]   2> 10941 WARN  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Interrupted
   [junit4]   2> java.lang.InterruptedException: null
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.reportInterruptAfterWait(AbstractQueuedSynchronizer.java:2037) ~[?:?]
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2071) ~[?:?]
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:189) [java/:?]
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:844) [?:?]
   [junit4]   2> 10941 DEBUG (TEST-TestSimTriggerIntegration.testNodeMarkersRegistration-seed#[77FFAC40FA650924]) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 10942 INFO  (TEST-TestSimTriggerIntegration.testNodeMarkersRegistration-seed#[77FFAC40FA650924]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testNodeMarkersRegistration
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestSimTriggerIntegration -Dtests.method=testNodeMarkersRegistration -Dtests.seed=77FFAC40FA650924 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.badapples=true -Dtests.locale=bs-Cyrl -Dtests.timezone=Pacific/Tahiti -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE 1.32s J0 | TestSimTriggerIntegration.testNodeMarkersRegistration <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: Path /autoscaling/nodeAdded/127.0.0.1:10055_solr should have been deleted
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([77FFAC40FA650924:6F45244CF450C4CB]:0)
   [junit4]    > 	at org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration.testNodeMarkersRegistration(TestSimTriggerIntegration.java:891)
   [junit4]    > 	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]    > 	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]    > 	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]    > 	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
   [junit4]    > 	at java.base/java.lang.Thread.run(Thread.java:844)
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/workspace/Lucene-Solr-BadApples-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.sim.TestSimTriggerIntegration_77FFAC40FA650924-001
   [junit4]   2> NOTE: test params are: codec=Lucene70, sim=RandomSimilarity(queryNorm=true): {}, locale=bs-Cyrl, timezone=Pacific/Tahiti
   [junit4]   2> NOTE: Linux 4.15.0-42-generic amd64/Oracle Corporation 9.0.4 (64-bit)/cpus=8,threads=1,free=506036992,total=536870912
   [junit4]   2> NOTE: All tests run in this JVM: [TestSimTriggerIntegration, TestSimTriggerIntegration]
   [junit4] Completed [5/5 (3!)] on J0 in 3.52s, 11 tests, 1 failure, 3 skipped <<< FAILURES!

[...truncated 13 lines...]
BUILD FAILED
/home/jenkins/workspace/Lucene-Solr-BadApples-7.x-Linux/lucene/common-build.xml:1572: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-BadApples-7.x-Linux/lucene/common-build.xml:1099: There were test failures: 5 suites, 55 tests, 3 failures, 15 ignored (15 assumptions) [seed: 77FFAC40FA650924]

Total time: 13 seconds

[repro] Setting last failure code to 256

[repro] Failures:
[repro]   0/5 failed: org.apache.solr.client.solrj.io.stream.StreamDecoratorTest
[repro]   3/5 failed: org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration
[repro] Exiting with code 256
+ mv lucene/build lucene/build.repro
+ mv solr/build solr/build.repro
+ mv lucene/build.orig lucene/build
+ mv solr/build.orig solr/build
Archiving artifacts
Setting ANT_1_8_2_HOME=/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
[WARNINGS] Parsing warnings in console log with parser Java Compiler (javac)
Setting ANT_1_8_2_HOME=/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting ANT_1_8_2_HOME=/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
<Git Blamer> Using GitBlamer to create author and commit information for all warnings.
<Git Blamer> GIT_COMMIT=bcf1a4eaf8d3b945f11833c9dfed4a17a6d6ac58, workspace=/var/lib/jenkins/workspace/Lucene-Solr-BadApples-7.x-Linux
[WARNINGS] Computing warning deltas based on reference build #137
Recording test results
Setting ANT_1_8_2_HOME=/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)
Setting ANT_1_8_2_HOME=/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting ANT_1_8_2_HOME=/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting ANT_1_8_2_HOME=/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting ANT_1_8_2_HOME=/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2

[JENKINS] Lucene-Solr-BadApples-7.x-Linux (32bit/jdk1.8.0_172) - Build # 139 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-BadApples-7.x-Linux/139/
Java: 32bit/jdk1.8.0_172 -server -XX:+UseSerialGC

1 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.update.processor.ScriptEngineTest

Error Message:
The test or suite printed 10008 bytes to stdout and stderr, even though the limit was set to 8192 bytes. Increase the limit with @Limit, ignore it completely with @SuppressSysoutChecks or run with -Dtests.verbose=true

Stack Trace:
java.lang.AssertionError: The test or suite printed 10008 bytes to stdout and stderr, even though the limit was set to 8192 bytes. Increase the limit with @Limit, ignore it completely with @SuppressSysoutChecks or run with -Dtests.verbose=true
	at __randomizedtesting.SeedInfo.seed([EE1EE3B3FE2CB4D]:0)
	at org.apache.lucene.util.TestRuleLimitSysouts.afterIfSuccessful(TestRuleLimitSysouts.java:211)
	at com.carrotsearch.randomizedtesting.rules.TestRuleAdapter$1.afterIfSuccessful(TestRuleAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:37)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 13591 lines...]
   [junit4] Suite: org.apache.solr.update.processor.ScriptEngineTest
   [junit4] IGNOR/A 0.00s J2 | ScriptEngineTest.testJRuby
   [junit4]    > Assumption #1: got: <[null]>, expected: every item is not null
   [junit4]   2> 662234 ERROR (Finalizer) [    ] o.a.s.c.CoreContainer CoreContainer was not close prior to finalize(), indicates a bug -- POSSIBLE RESOURCE LEAK!!!  instance=13677481
   [junit4]   2> 662234 ERROR (Finalizer) [    ] o.a.s.c.CoreContainer CoreContainer was not close prior to finalize(), indicates a bug -- POSSIBLE RESOURCE LEAK!!!  instance=14650409
   [junit4]   2> 662235 ERROR (Finalizer) [    ] o.a.s.c.CoreContainer CoreContainer was not close prior to finalize(), indicates a bug -- POSSIBLE RESOURCE LEAK!!!  instance=2700282
   [junit4]   2> 662235 ERROR (Finalizer) [    ] o.a.s.c.CoreContainer CoreContainer was not close prior to finalize(), indicates a bug -- POSSIBLE RESOURCE LEAK!!!  instance=18086052
   [junit4]   2> 662235 ERROR (Finalizer) [    ] o.a.s.c.CoreContainer CoreContainer was not close prior to finalize(), indicates a bug -- POSSIBLE RESOURCE LEAK!!!  instance=7162900
   [junit4]   2> 662235 ERROR (Finalizer) [    ] o.a.s.c.CoreContainer CoreContainer was not close prior to finalize(), indicates a bug -- POSSIBLE RESOURCE LEAK!!!  instance=11874932
   [junit4]   2> 662235 ERROR (Finalizer) [    ] o.a.s.c.CoreContainer CoreContainer was not close prior to finalize(), indicates a bug -- POSSIBLE RESOURCE LEAK!!!  instance=21786427
   [junit4]   2> 662235 ERROR (Finalizer) [    ] o.a.s.c.CoreContainer CoreContainer was not close prior to finalize(), indicates a bug -- POSSIBLE RESOURCE LEAK!!!  instance=17981678
   [junit4]   2> 662235 ERROR (Finalizer) [    ] o.a.s.c.CoreContainer CoreContainer was not close prior to finalize(), indicates a bug -- POSSIBLE RESOURCE LEAK!!!  instance=22122932
   [junit4]   2> 662235 ERROR (Finalizer) [    ] o.a.s.c.CoreContainer CoreContainer was not close prior to finalize(), indicates a bug -- POSSIBLE RESOURCE LEAK!!!  instance=10239032
   [junit4]   2> 662235 ERROR (Finalizer) [    ] o.a.s.c.CoreContainer CoreContainer was not close prior to finalize(), indicates a bug -- POSSIBLE RESOURCE LEAK!!!  instance=13089447
   [junit4]   2> 662236 ERROR (Finalizer) [    ] o.a.s.c.CoreContainer CoreContainer was not close prior to finalize(), indicates a bug -- POSSIBLE RESOURCE LEAK!!!  instance=12891777
   [junit4]   2> 662236 ERROR (Finalizer) [    ] o.a.s.c.CoreContainer CoreContainer was not close prior to finalize(), indicates a bug -- POSSIBLE RESOURCE LEAK!!!  instance=23023644
   [junit4]   2> 662236 ERROR (Finalizer) [    ] o.a.s.c.CoreContainer CoreContainer was not close prior to finalize(), indicates a bug -- POSSIBLE RESOURCE LEAK!!!  instance=14930483
   [junit4]   2> 662236 ERROR (Finalizer) [    ] o.a.s.c.CoreContainer CoreContainer was not close prior to finalize(), indicates a bug -- POSSIBLE RESOURCE LEAK!!!  instance=25319882
   [junit4]   2> 662236 ERROR (Finalizer) [    ] o.a.s.c.CoreContainer CoreContainer was not close prior to finalize(), indicates a bug -- POSSIBLE RESOURCE LEAK!!!  instance=9404997
   [junit4]   2> 662236 ERROR (Finalizer) [    ] o.a.s.c.CoreContainer CoreContainer was not close prior to finalize(), indicates a bug -- POSSIBLE RESOURCE LEAK!!!  instance=28702712
   [junit4]   2> 662236 ERROR (Finalizer) [    ] o.a.s.c.CoreContainer CoreContainer was not close prior to finalize(), indicates a bug -- POSSIBLE RESOURCE LEAK!!!  instance=17287479
   [junit4]   2> 662236 ERROR (Finalizer) [    ] o.a.s.c.CoreContainer CoreContainer was not close prior to finalize(), indicates a bug -- POSSIBLE RESOURCE LEAK!!!  instance=16028244
   [junit4]   2> 662236 ERROR (Finalizer) [    ] o.a.s.c.CoreContainer CoreContainer was not close prior to finalize(), indicates a bug -- POSSIBLE RESOURCE LEAK!!!  instance=12954853
   [junit4]   2> 662236 ERROR (Finalizer) [    ] o.a.s.c.CoreContainer CoreContainer was not close prior to finalize(), indicates a bug -- POSSIBLE RESOURCE LEAK!!!  instance=5606898
   [junit4]   2> 662236 ERROR (Finalizer) [    ] o.a.s.c.CoreContainer CoreContainer was not close prior to finalize(), indicates a bug -- POSSIBLE RESOURCE LEAK!!!  instance=30457150
   [junit4]   2> 662236 ERROR (Finalizer) [    ] o.a.s.c.CoreContainer CoreContainer was not close prior to finalize(), indicates a bug -- POSSIBLE RESOURCE LEAK!!!  instance=22609826
   [junit4]   2> 662236 ERROR (Finalizer) [    ] o.a.s.c.CoreContainer CoreContainer was not close prior to finalize(), indicates a bug -- POSSIBLE RESOURCE LEAK!!!  instance=3265256
   [junit4]   2> 662236 ERROR (Finalizer) [    ] o.a.s.c.CoreContainer CoreContainer was not close prior to finalize(), indicates a bug -- POSSIBLE RESOURCE LEAK!!!  instance=14411585
   [junit4]   2> 662236 ERROR (Finalizer) [    ] o.a.s.c.CoreContainer CoreContainer was not close prior to finalize(), indicates a bug -- POSSIBLE RESOURCE LEAK!!!  instance=13671273
   [junit4]   2> 662237 ERROR (Finalizer) [    ] o.a.s.c.CoreContainer CoreContainer was not close prior to finalize(), indicates a bug -- POSSIBLE RESOURCE LEAK!!!  instance=15458110
   [junit4]   2> 662237 ERROR (Finalizer) [    ] o.a.s.c.CoreContainer CoreContainer was not close prior to finalize(), indicates a bug -- POSSIBLE RESOURCE LEAK!!!  instance=5371545
   [junit4]   2> 662237 ERROR (Finalizer) [    ] o.a.s.c.CoreContainer CoreContainer was not close prior to finalize(), indicates a bug -- POSSIBLE RESOURCE LEAK!!!  instance=19785341
   [junit4]   2> 662237 ERROR (Finalizer) [    ] o.a.s.c.CoreContainer CoreContainer was not close prior to finalize(), indicates a bug -- POSSIBLE RESOURCE LEAK!!!  instance=30552866
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=826, maxMBSortInHeap=5.131145407085953, sim=RandomSimilarity(queryNorm=false): {}, locale=is-IS, timezone=America/Anchorage
   [junit4]   2> NOTE: Linux 4.15.0-42-generic i386/Oracle Corporation 1.8.0_172 (32-bit)/cpus=8,threads=1,free=250838560,total=518979584
   [junit4]   2> NOTE: All tests run in this JVM: [StatelessScriptUpdateProcessorFactoryTest, PeerSyncWithLeaderAndIndexFingerprintCachingTest, TestGraphTermsQParserPlugin, TestSimNodeAddedTrigger, ChaosMonkeySafeLeaderTest, UtilsToolTest, TestPartialUpdateDeduplication, ForceLeaderTest, TestBinaryField, HdfsDirectoryFactoryTest, CollectionsAPISolrJTest, AddReplicaTest, RandomizedTaggerTest, TestExceedMaxTermLength, SolrJmxReporterCloudTest, MultiThreadedOCPTest, SuggestComponentTest, UninvertDocValuesMergePolicyTest, TestSimpleTextCodec, TestCSVResponseWriter, ClassificationUpdateProcessorIntegrationTest, ParsingFieldUpdateProcessorsTest, DistributedFacetExistsSmallTest, TestCloudNestedDocsSort, TestFacetMethods, ShardRoutingCustomTest, NodeMarkersRegistrationTest, TestHalfAndHalfDocValues, SpellCheckComponentTest, QueryResultKeyTest, SolrCoreMetricManagerTest, TestWriterPerf, BlockJoinFacetRandomTest, CursorMarkTest, ImplicitSnitchTest, TestFuzzyAnalyzedSuggestions, CdcrRequestHandlerTest, TestSha256AuthenticationProvider, SuggesterFSTTest, TestHttpShardHandlerFactory, SpellCheckCollatorTest, CdcrBootstrapTest, TestDynamicFieldCollectionResource, DistributedIntervalFacetingTest, HdfsUnloadDistributedZkTest, TestPayloadCheckQParserPlugin, NumericFieldsTest, TestSubQueryTransformerDistrib, TestComplexPhraseLeadingWildcard, TestMinMaxOnMultiValuedField, TestSearchPerf, TestConfigSetImmutable, HdfsSyncSliceTest, TestRecoveryHdfs, ExternalFileFieldSortTest, SuggesterWFSTTest, RollingRestartTest, DateFieldTest, TestAuthenticationFramework, DirectSolrSpellCheckerTest, V2ApiIntegrationTest, TestInitQParser, TestSchemaVersionResource, TestJettySolrRunner, DistributedTermsComponentTest, SolrCloudReportersTest, TestMergePolicyConfig, BadCopyFieldTest, TestDistribDocBasedVersion, DistributedQueryElevationComponentTest, ByteBuffersDirectoryFactoryTest, SolrTestCaseJ4Test, TestCloudSchemaless, TestFieldResource, ConnectionManagerTest, TestRemoteStreaming, URLClassifyProcessorTest, ResourceLoaderTest, QueryEqualityTest, TestExtendedDismaxParser, SolrXmlInZkTest, HdfsDirectoryTest, TriggerSetPropertiesIntegrationTest, TestWithCollection, TestPullReplica, TestFilteredDocIdSet, SpatialFilterTest, OverseerTest, ScriptEngineTest]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ScriptEngineTest -Dtests.seed=EE1EE3B3FE2CB4D -Dtests.multiplier=3 -Dtests.slow=true -Dtests.badapples=true -Dtests.locale=is-IS -Dtests.timezone=America/Anchorage -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   0.00s J2 | ScriptEngineTest (suite) <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: The test or suite printed 10008 bytes to stdout and stderr, even though the limit was set to 8192 bytes. Increase the limit with @Limit, ignore it completely with @SuppressSysoutChecks or run with -Dtests.verbose=true
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([EE1EE3B3FE2CB4D]:0)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4] Completed [264/836 (1!)] on J2 in 0.26s, 6 tests, 1 failure, 1 skipped <<< FAILURES!

[...truncated 46904 lines...]
[repro] Jenkins log URL: https://jenkins.thetaphi.de/job/Lucene-Solr-BadApples-7.x-Linux/139/consoleText

[repro] Revision: bcf1a4eaf8d3b945f11833c9dfed4a17a6d6ac58

[repro] Repro line:  ant test  -Dtestcase=ScriptEngineTest -Dtests.seed=EE1EE3B3FE2CB4D -Dtests.multiplier=3 -Dtests.slow=true -Dtests.badapples=true -Dtests.locale=is-IS -Dtests.timezone=America/Anchorage -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1

[repro] ant clean

[...truncated 6 lines...]
[repro] Test suites by module:
[repro]    solr/core
[repro]       ScriptEngineTest
[repro] ant compile-test

[...truncated 3605 lines...]
[repro] ant test-nocompile -Dtests.dups=5 -Dtests.maxfailures=5 -Dtests.class="*.ScriptEngineTest" -Dtests.showOutput=onerror  -Dtests.seed=EE1EE3B3FE2CB4D -Dtests.multiplier=3 -Dtests.slow=true -Dtests.badapples=true -Dtests.locale=is-IS -Dtests.timezone=America/Anchorage -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1

[...truncated 100 lines...]
[repro] Failures:
[repro]   0/5 failed: org.apache.solr.update.processor.ScriptEngineTest
[repro] Exiting with code 0

[...truncated 40 lines...]