You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2018/03/07 09:25:42 UTC

[JENKINS] Lucene-Solr-BadApples-NightlyTests-7.x - Build # 1 - Unstable

Build: https://builds.apache.org/job/Lucene-Solr-BadApples-NightlyTests-7.x/1/

14 tests failed.
FAILED:  org.apache.solr.ltr.TestLTRReRankingPipeline.testDifferentTopN

Error Message:
expected:<1.0> but was:<0.0>

Stack Trace:
java.lang.AssertionError: expected:<1.0> but was:<0.0>
	at __randomizedtesting.SeedInfo.seed([A35BB49AA5253521:52FAC6CA909EFFB3]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:443)
	at org.junit.Assert.assertEquals(Assert.java:512)
	at org.apache.solr.ltr.TestLTRReRankingPipeline.testDifferentTopN(TestLTRReRankingPipeline.java:255)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.ZkControllerTest.testPublishAndWaitForDownStates

Error Message:
The ZkController.publishAndWaitForDownStates should have timed out but it didn't

Stack Trace:
java.lang.AssertionError: The ZkController.publishAndWaitForDownStates should have timed out but it didn't
	at __randomizedtesting.SeedInfo.seed([4CD29D80308A9012:6BDDF16C911DDF2D]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.cloud.ZkControllerTest.testPublishAndWaitForDownStates(ZkControllerTest.java:306)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.testSearchRate

Error Message:
The trigger did not fire at all

Stack Trace:
java.lang.AssertionError: The trigger did not fire at all
	at __randomizedtesting.SeedInfo.seed([4CD29D80308A9012:119A8309FF4C365D]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.testSearchRate(TestLargeCluster.java:548)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.testAddNode

Error Message:
no MOVEREPLICA ops?

Stack Trace:
java.lang.AssertionError: no MOVEREPLICA ops?
	at __randomizedtesting.SeedInfo.seed([4CD29D80308A9012:EB3D8023FFC71F0A]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.testAddNode(TestLargeCluster.java:262)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testSearchRate

Error Message:
The trigger did not fire at all

Stack Trace:
java.lang.AssertionError: The trigger did not fire at all
	at __randomizedtesting.SeedInfo.seed([4CD29D80308A9012:119A8309FF4C365D]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testSearchRate(TestTriggerIntegration.java:1184)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeMarkersRegistration

Error Message:
Path /autoscaling/nodeAdded/127.0.0.1:10014_solr should have been deleted

Stack Trace:
java.lang.AssertionError: Path /autoscaling/nodeAdded/127.0.0.1:10014_solr should have been deleted
	at __randomizedtesting.SeedInfo.seed([4CD29D80308A9012:5468158C3EBF5DFD]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertFalse(Assert.java:68)
	at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeMarkersRegistration(TestTriggerIntegration.java:846)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsBasicDistributedZkTest

Error Message:
ObjectTracker found 1 object(s) that were not released!!! [NRTCachingDirectory] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.NRTCachingDirectory  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)  at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:351)  at org.apache.solr.core.SolrCore.cleanupOldIndexDirectories(SolrCore.java:3031)  at org.apache.solr.core.SolrCore.close(SolrCore.java:1604)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1000)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:863)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1040)  at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:640)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  

Stack Trace:
java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [NRTCachingDirectory]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.NRTCachingDirectory
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
	at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:351)
	at org.apache.solr.core.SolrCore.cleanupOldIndexDirectories(SolrCore.java:3031)
	at org.apache.solr.core.SolrCore.close(SolrCore.java:1604)
	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1000)
	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:863)
	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1040)
	at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:640)
	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)


	at __randomizedtesting.SeedInfo.seed([4CD29D80308A9012]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNull(Assert.java:551)
	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:297)
	at sun.reflect.GeneratedMethodAccessor69.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:897)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest.test

Error Message:
expected:<0> but was:<1>

Stack Trace:
java.lang.AssertionError: expected:<0> but was:<1>
	at __randomizedtesting.SeedInfo.seed([4CD29D80308A9012:C486A25A9E76FDEA]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test(ChaosMonkeySafeLeaderTest.java:147)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.core.TestJmxIntegration.testJmxOnCoreReload

Error Message:
Number of registered MBeans is not the same as info registry size expected:<108> but was:<0>

Stack Trace:
java.lang.AssertionError: Number of registered MBeans is not the same as info registry size expected:<108> but was:<0>
	at __randomizedtesting.SeedInfo.seed([4CD29D80308A9012:4E6B46EC43C94CDF]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.apache.solr.core.TestJmxIntegration.testJmxOnCoreReload(TestJmxIntegration.java:180)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload

Error Message:
expected:<[{indexVersion=1520406312516,generation=2,filelist=[_3x.fdt, _3x.fdx, _3x.fnm, _3x.nvd, _3x.nvm, _3x.si, _3x_Asserting_0.doc, _3x_Asserting_0.tim, _3x_Asserting_0.tip, _70.fdt, _70.fdx, _70.fnm, _70.nvd, _70.nvm, _70.si, _70_Asserting_0.doc, _70_Asserting_0.tim, _70_Asserting_0.tip, _7a.cfe, _7a.cfs, _7a.si, _7l.cfe, _7l.cfs, _7l.si, _7w.cfe, _7w.cfs, _7w.si, _87.cfe, _87.cfs, _87.si, _8i.cfe, _8i.cfs, _8i.si, _8t.cfe, _8t.cfs, _8t.si, _94.cfe, _94.cfs, _94.si, _9f.cfe, _9f.cfs, _9f.si, _9g.fdt, _9g.fdx, _9g.fnm, _9g.nvd, _9g.nvm, _9g.si, _9g_Asserting_0.doc, _9g_Asserting_0.tim, _9g_Asserting_0.tip, _9h.fdt, _9h.fdx, _9h.fnm, _9h.nvd, _9h.nvm, _9h.si, _9h_Asserting_0.doc, _9h_Asserting_0.tim, _9h_Asserting_0.tip, _9i.fdt, _9i.fdx, _9i.fnm, _9i.nvd, _9i.nvm, _9i.si, _9i_Asserting_0.doc, _9i_Asserting_0.tim, _9i_Asserting_0.tip, _9j.fdt, _9j.fdx, _9j.fnm, _9j.nvd, _9j.nvm, _9j.si, _9j_Asserting_0.doc, _9j_Asserting_0.tim, _9j_Asserting_0.tip, _9k.fdt, _9k.fdx, _9k.fnm, _9k.nvd, _9k.nvm, _9k.si, _9k_Asserting_0.doc, _9k_Asserting_0.tim, _9k_Asserting_0.tip, _9l.fdt, _9l.fdx, _9l.fnm, _9l.nvd, _9l.nvm, _9l.si, _9l_Asserting_0.doc, _9l_Asserting_0.tim, _9l_Asserting_0.tip, _9m.fdt, _9m.fdx, _9m.fnm, _9m.nvd, _9m.nvm, _9m.si, _9m_Asserting_0.doc, _9m_Asserting_0.tim, _9m_Asserting_0.tip, _9n.fdt, _9n.fdx, _9n.fnm, _9n.nvd, _9n.nvm, _9n.si, _9n_Asserting_0.doc, _9n_Asserting_0.tim, _9n_Asserting_0.tip, _9o.fdt, _9o.fdx, _9o.fnm, _9o.nvd, _9o.nvm, _9o.si, _9o_Asserting_0.doc, _9o_Asserting_0.tim, _9o_Asserting_0.tip, _9p.fdt, _9p.fdx, _9p.fnm, _9p.nvd, _9p.nvm, _9p.si, _9p_Asserting_0.doc, _9p_Asserting_0.tim, _9p_Asserting_0.tip, segments_2]}]> but was:<[{indexVersion=1520406312516,generation=2,filelist=[_3x.fdt, _3x.fdx, _3x.fnm, _3x.nvd, _3x.nvm, _3x.si, _3x_Asserting_0.doc, _3x_Asserting_0.tim, _3x_Asserting_0.tip, _70.fdt, _70.fdx, _70.fnm, _70.nvd, _70.nvm, _70.si, _70_Asserting_0.doc, _70_Asserting_0.tim, _70_Asserting_0.tip, _7a.cfe, _7a.cfs, _7a.si, _7l.cfe, _7l.cfs, _7l.si, _7w.cfe, _7w.cfs, _7w.si, _87.cfe, _87.cfs, _87.si, _8i.cfe, _8i.cfs, _8i.si, _8t.cfe, _8t.cfs, _8t.si, _94.cfe, _94.cfs, _94.si, _9f.cfe, _9f.cfs, _9f.si, _9g.fdt, _9g.fdx, _9g.fnm, _9g.nvd, _9g.nvm, _9g.si, _9g_Asserting_0.doc, _9g_Asserting_0.tim, _9g_Asserting_0.tip, _9h.fdt, _9h.fdx, _9h.fnm, _9h.nvd, _9h.nvm, _9h.si, _9h_Asserting_0.doc, _9h_Asserting_0.tim, _9h_Asserting_0.tip, _9i.fdt, _9i.fdx, _9i.fnm, _9i.nvd, _9i.nvm, _9i.si, _9i_Asserting_0.doc, _9i_Asserting_0.tim, _9i_Asserting_0.tip, _9j.fdt, _9j.fdx, _9j.fnm, _9j.nvd, _9j.nvm, _9j.si, _9j_Asserting_0.doc, _9j_Asserting_0.tim, _9j_Asserting_0.tip, _9k.fdt, _9k.fdx, _9k.fnm, _9k.nvd, _9k.nvm, _9k.si, _9k_Asserting_0.doc, _9k_Asserting_0.tim, _9k_Asserting_0.tip, _9l.fdt, _9l.fdx, _9l.fnm, _9l.nvd, _9l.nvm, _9l.si, _9l_Asserting_0.doc, _9l_Asserting_0.tim, _9l_Asserting_0.tip, _9m.fdt, _9m.fdx, _9m.fnm, _9m.nvd, _9m.nvm, _9m.si, _9m_Asserting_0.doc, _9m_Asserting_0.tim, _9m_Asserting_0.tip, _9n.fdt, _9n.fdx, _9n.fnm, _9n.nvd, _9n.nvm, _9n.si, _9n_Asserting_0.doc, _9n_Asserting_0.tim, _9n_Asserting_0.tip, _9o.fdt, _9o.fdx, _9o.fnm, _9o.nvd, _9o.nvm, _9o.si, _9o_Asserting_0.doc, _9o_Asserting_0.tim, _9o_Asserting_0.tip, _9p.fdt, _9p.fdx, _9p.fnm, _9p.nvd, _9p.nvm, _9p.si, _9p_Asserting_0.doc, _9p_Asserting_0.tim, _9p_Asserting_0.tip, segments_2]}, {indexVersion=1520406312516,generation=3,filelist=[_3x.fdt, _3x.fdx, _3x.fnm, _3x.nvd, _3x.nvm, _3x.si, _3x_Asserting_0.doc, _3x_Asserting_0.tim, _3x_Asserting_0.tip, _70.fdt, _70.fdx, _70.fnm, _70.nvd, _70.nvm, _70.si, _70_Asserting_0.doc, _70_Asserting_0.tim, _70_Asserting_0.tip, _7a.cfe, _7a.cfs, _7a.si, _7l.cfe, _7l.cfs, _7l.si, _7w.cfe, _7w.cfs, _7w.si, _87.cfe, _87.cfs, _87.si, _8i.cfe, _8i.cfs, _8i.si, _8t.cfe, _8t.cfs, _8t.si, _94.cfe, _94.cfs, _94.si, _9f.cfe, _9f.cfs, _9f.si, _9q.cfe, _9q.cfs, _9q.si, segments_3]}]>

Stack Trace:
java.lang.AssertionError: expected:<[{indexVersion=1520406312516,generation=2,filelist=[_3x.fdt, _3x.fdx, _3x.fnm, _3x.nvd, _3x.nvm, _3x.si, _3x_Asserting_0.doc, _3x_Asserting_0.tim, _3x_Asserting_0.tip, _70.fdt, _70.fdx, _70.fnm, _70.nvd, _70.nvm, _70.si, _70_Asserting_0.doc, _70_Asserting_0.tim, _70_Asserting_0.tip, _7a.cfe, _7a.cfs, _7a.si, _7l.cfe, _7l.cfs, _7l.si, _7w.cfe, _7w.cfs, _7w.si, _87.cfe, _87.cfs, _87.si, _8i.cfe, _8i.cfs, _8i.si, _8t.cfe, _8t.cfs, _8t.si, _94.cfe, _94.cfs, _94.si, _9f.cfe, _9f.cfs, _9f.si, _9g.fdt, _9g.fdx, _9g.fnm, _9g.nvd, _9g.nvm, _9g.si, _9g_Asserting_0.doc, _9g_Asserting_0.tim, _9g_Asserting_0.tip, _9h.fdt, _9h.fdx, _9h.fnm, _9h.nvd, _9h.nvm, _9h.si, _9h_Asserting_0.doc, _9h_Asserting_0.tim, _9h_Asserting_0.tip, _9i.fdt, _9i.fdx, _9i.fnm, _9i.nvd, _9i.nvm, _9i.si, _9i_Asserting_0.doc, _9i_Asserting_0.tim, _9i_Asserting_0.tip, _9j.fdt, _9j.fdx, _9j.fnm, _9j.nvd, _9j.nvm, _9j.si, _9j_Asserting_0.doc, _9j_Asserting_0.tim, _9j_Asserting_0.tip, _9k.fdt, _9k.fdx, _9k.fnm, _9k.nvd, _9k.nvm, _9k.si, _9k_Asserting_0.doc, _9k_Asserting_0.tim, _9k_Asserting_0.tip, _9l.fdt, _9l.fdx, _9l.fnm, _9l.nvd, _9l.nvm, _9l.si, _9l_Asserting_0.doc, _9l_Asserting_0.tim, _9l_Asserting_0.tip, _9m.fdt, _9m.fdx, _9m.fnm, _9m.nvd, _9m.nvm, _9m.si, _9m_Asserting_0.doc, _9m_Asserting_0.tim, _9m_Asserting_0.tip, _9n.fdt, _9n.fdx, _9n.fnm, _9n.nvd, _9n.nvm, _9n.si, _9n_Asserting_0.doc, _9n_Asserting_0.tim, _9n_Asserting_0.tip, _9o.fdt, _9o.fdx, _9o.fnm, _9o.nvd, _9o.nvm, _9o.si, _9o_Asserting_0.doc, _9o_Asserting_0.tim, _9o_Asserting_0.tip, _9p.fdt, _9p.fdx, _9p.fnm, _9p.nvd, _9p.nvm, _9p.si, _9p_Asserting_0.doc, _9p_Asserting_0.tim, _9p_Asserting_0.tip, segments_2]}]> but was:<[{indexVersion=1520406312516,generation=2,filelist=[_3x.fdt, _3x.fdx, _3x.fnm, _3x.nvd, _3x.nvm, _3x.si, _3x_Asserting_0.doc, _3x_Asserting_0.tim, _3x_Asserting_0.tip, _70.fdt, _70.fdx, _70.fnm, _70.nvd, _70.nvm, _70.si, _70_Asserting_0.doc, _70_Asserting_0.tim, _70_Asserting_0.tip, _7a.cfe, _7a.cfs, _7a.si, _7l.cfe, _7l.cfs, _7l.si, _7w.cfe, _7w.cfs, _7w.si, _87.cfe, _87.cfs, _87.si, _8i.cfe, _8i.cfs, _8i.si, _8t.cfe, _8t.cfs, _8t.si, _94.cfe, _94.cfs, _94.si, _9f.cfe, _9f.cfs, _9f.si, _9g.fdt, _9g.fdx, _9g.fnm, _9g.nvd, _9g.nvm, _9g.si, _9g_Asserting_0.doc, _9g_Asserting_0.tim, _9g_Asserting_0.tip, _9h.fdt, _9h.fdx, _9h.fnm, _9h.nvd, _9h.nvm, _9h.si, _9h_Asserting_0.doc, _9h_Asserting_0.tim, _9h_Asserting_0.tip, _9i.fdt, _9i.fdx, _9i.fnm, _9i.nvd, _9i.nvm, _9i.si, _9i_Asserting_0.doc, _9i_Asserting_0.tim, _9i_Asserting_0.tip, _9j.fdt, _9j.fdx, _9j.fnm, _9j.nvd, _9j.nvm, _9j.si, _9j_Asserting_0.doc, _9j_Asserting_0.tim, _9j_Asserting_0.tip, _9k.fdt, _9k.fdx, _9k.fnm, _9k.nvd, _9k.nvm, _9k.si, _9k_Asserting_0.doc, _9k_Asserting_0.tim, _9k_Asserting_0.tip, _9l.fdt, _9l.fdx, _9l.fnm, _9l.nvd, _9l.nvm, _9l.si, _9l_Asserting_0.doc, _9l_Asserting_0.tim, _9l_Asserting_0.tip, _9m.fdt, _9m.fdx, _9m.fnm, _9m.nvd, _9m.nvm, _9m.si, _9m_Asserting_0.doc, _9m_Asserting_0.tim, _9m_Asserting_0.tip, _9n.fdt, _9n.fdx, _9n.fnm, _9n.nvd, _9n.nvm, _9n.si, _9n_Asserting_0.doc, _9n_Asserting_0.tim, _9n_Asserting_0.tip, _9o.fdt, _9o.fdx, _9o.fnm, _9o.nvd, _9o.nvm, _9o.si, _9o_Asserting_0.doc, _9o_Asserting_0.tim, _9o_Asserting_0.tip, _9p.fdt, _9p.fdx, _9p.fnm, _9p.nvd, _9p.nvm, _9p.si, _9p_Asserting_0.doc, _9p_Asserting_0.tim, _9p_Asserting_0.tip, segments_2]}, {indexVersion=1520406312516,generation=3,filelist=[_3x.fdt, _3x.fdx, _3x.fnm, _3x.nvd, _3x.nvm, _3x.si, _3x_Asserting_0.doc, _3x_Asserting_0.tim, _3x_Asserting_0.tip, _70.fdt, _70.fdx, _70.fnm, _70.nvd, _70.nvm, _70.si, _70_Asserting_0.doc, _70_Asserting_0.tim, _70_Asserting_0.tip, _7a.cfe, _7a.cfs, _7a.si, _7l.cfe, _7l.cfs, _7l.si, _7w.cfe, _7w.cfs, _7w.si, _87.cfe, _87.cfs, _87.si, _8i.cfe, _8i.cfs, _8i.si, _8t.cfe, _8t.cfs, _8t.si, _94.cfe, _94.cfs, _94.si, _9f.cfe, _9f.cfs, _9f.si, _9q.cfe, _9q.cfs, _9q.si, segments_3]}]>
	at __randomizedtesting.SeedInfo.seed([4CD29D80308A9012:690586B040C29E11]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:147)
	at org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload(TestReplicationHandler.java:1280)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestIndexFetchOnMasterRestart

Error Message:
IOException occured when talking to server at: http://127.0.0.1:33407/solr/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:33407/solr/collection1
	at __randomizedtesting.SeedInfo.seed([4CD29D80308A9012:942559649B51524E]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:657)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:173)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:138)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:152)
	at org.apache.solr.handler.TestReplicationHandler.index(TestReplicationHandler.java:180)
	at org.apache.solr.handler.TestReplicationHandler.doTestIndexFetchOnMasterRestart(TestReplicationHandler.java:643)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:33407 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:141)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:542)
	... 47 more


FAILED:  org.apache.solr.handler.admin.AutoscalingHistoryHandlerTest.testHistory

Error Message:
replica never fully recovered

Stack Trace:
java.lang.AssertionError: replica never fully recovered
	at __randomizedtesting.SeedInfo.seed([4CD29D80308A9012:212E397D8AC26F15]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.handler.admin.AutoscalingHistoryHandlerTest.waitForRecovery(AutoscalingHistoryHandlerTest.java:405)
	at org.apache.solr.handler.admin.AutoscalingHistoryHandlerTest.testHistory(AutoscalingHistoryHandlerTest.java:341)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.uninverting.TestDocTermOrds.testTriggerUnInvertLimit

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
	at __randomizedtesting.SeedInfo.seed([4CD29D80308A9012]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.uninverting.TestDocTermOrds

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
	at __randomizedtesting.SeedInfo.seed([4CD29D80308A9012]:0)




Build Log:
[...truncated 12854 lines...]
   [junit4] Suite: org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration
   [junit4]   2> 126425 INFO  (SUITE-TestTriggerIntegration-seed#[4CD29D80308A9012]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.sim.TestTriggerIntegration_4CD29D80308A9012-001/init-core-data-001
   [junit4]   2> 126440 WARN  (SUITE-TestTriggerIntegration-seed#[4CD29D80308A9012]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=2 numCloses=2
   [junit4]   2> 126440 INFO  (SUITE-TestTriggerIntegration-seed#[4CD29D80308A9012]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 126442 INFO  (SUITE-TestTriggerIntegration-seed#[4CD29D80308A9012]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 126713 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 126713 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 0
   [junit4]   2> 126714 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10000_solr]
   [junit4]   2> 126714 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion -1
   [junit4]   2> 126714 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 0
   [junit4]   2> 126751 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 126751 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion 0
   [junit4]   2> 126807 DEBUG (ScheduledTrigger-100-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 126938 DEBUG (ScheduledTrigger-100-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 126973 DEBUG (ScheduledTrigger-100-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 127011 DEBUG (ScheduledTrigger-100-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 127083 DEBUG (ScheduledTrigger-100-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 127100 INFO  (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[4CD29D80308A9012]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testNodeLostTrigger
   [junit4]   2> 127100 DEBUG (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[4CD29D80308A9012]) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
   [junit4]   2> 127102 INFO  (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[4CD29D80308A9012]) [    ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
   [junit4]   2> 127119 DEBUG (ScheduledTrigger-100-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 127120 WARN  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 127121 DEBUG (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[4CD29D80308A9012]) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 127168 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 127168 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2
   [junit4]   2> 127169 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10000_solr]
   [junit4]   2> 127169 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion -1
   [junit4]   2> 127169 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
   [junit4]   2> 127188 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 127188 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
   [junit4]   2> 127210 DEBUG (ScheduledTrigger-103-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 127230 DEBUG (ScheduledTrigger-103-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 127716 DEBUG (ScheduledTrigger-103-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 127749 DEBUG (ScheduledTrigger-103-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 127771 DEBUG (ScheduledTrigger-103-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 127806 DEBUG (ScheduledTrigger-103-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 127826 DEBUG (ScheduledTrigger-103-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 127828 DEBUG (simCloudManagerPool-102-thread-1) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 127829 DEBUG (simCloudManagerPool-102-thread-1) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 3
   [junit4]   2> 127842 DEBUG (simCloudManagerPool-102-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10000_solr]
   [junit4]   2> 127843 DEBUG (simCloudManagerPool-102-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10000_solr]
   [junit4]   2> 127843 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
   [junit4]   2> 127843 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 127843 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 127843 DEBUG (ScheduledTrigger-103-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2
   [junit4]   2> 127843 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
   [junit4]   2> 127863 DEBUG (ScheduledTrigger-103-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
   [junit4]   2> 127863 DEBUG (ScheduledTrigger-103-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10000_solr
   [junit4]   2> 127863 DEBUG (ScheduledTrigger-103-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 1
   [junit4]   2> 127863 DEBUG (ScheduledTrigger-103-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10000_solr
   [junit4]   2> 127899 DEBUG (ScheduledTrigger-103-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
   [junit4]   2> 127899 DEBUG (ScheduledTrigger-103-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 1
   [junit4]   2> 127900 DEBUG (ScheduledTrigger-103-thread-2) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:10000_solr]
   [junit4]   2> 127900 DEBUG (ScheduledTrigger-103-thread-2) [    ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"9bce9a568f364Tbtfp7k0wdqt33lvv72jd5a5p8",
   [junit4]   2>   "source":"node_lost_trigger",
   [junit4]   2>   "eventTime":2740986478916452,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[2740986478916452],
   [junit4]   2>     "nodeNames":["127.0.0.1:10000_solr"]}}
   [junit4]   2> 127900 DEBUG (ScheduledTrigger-103-thread-2) [    ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_lost_trigger]
   [junit4]   2> 127991 DEBUG (AutoscalingActionExecutor-104-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"9bce9a568f364Tbtfp7k0wdqt33lvv72jd5a5p8",
   [junit4]   2>   "source":"node_lost_trigger",
   [junit4]   2>   "eventTime":2740986478916452,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[2740986478916452],
   [junit4]   2>     "_enqueue_time_":2740988359981252,
   [junit4]   2>     "nodeNames":["127.0.0.1:10000_solr"]}}
   [junit4]   2> 128049 DEBUG (AutoscalingActionExecutor-104-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms
   [junit4]   2> 128049 DEBUG (AutoscalingActionExecutor-104-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_lost_trigger after 100ms
   [junit4]   2> 128049 DEBUG (AutoscalingActionExecutor-104-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing took 2899 ms for event id=9bce9a568f364Tbtfp7k0wdqt33lvv72jd5a5p8
   [junit4]   2> 128192 DEBUG (ScheduledTrigger-103-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
   [junit4]   2> 128083 DEBUG (simCloudManagerPool-102-thread-4) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 128428 DEBUG (simCloudManagerPool-102-thread-4) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 4
   [junit4]   2> 128429 DEBUG (simCloudManagerPool-102-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr]
   [junit4]   2> 128429 DEBUG (simCloudManagerPool-102-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr]
   [junit4]   2> 128429 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 4
   [junit4]   2> 128429 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 128429 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion 4
   [junit4]   2> 128428 DEBUG (ScheduledTrigger-103-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 1
   [junit4]   2> 128428 DEBUG (ScheduledTrigger-103-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
   [junit4]   2> 128464 DEBUG (ScheduledTrigger-103-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 1
   [junit4]   2> 128465 DEBUG (ScheduledTrigger-103-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
   [junit4]   2> 128465 DEBUG (ScheduledTrigger-103-thread-3) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:10000_solr]
   [junit4]   2> 128465 DEBUG (ScheduledTrigger-103-thread-3) [    ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"9bce9a3d82d8aTbtfp7k0wdqt33lvv72jd5a5pb",
   [junit4]   2>   "source":".auto_add_replicas",
   [junit4]   2>   "eventTime":2740986452651402,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[2740986452651402],
   [junit4]   2>     "nodeNames":["127.0.0.1:10000_solr"]}}
   [junit4]   2> 128465 DEBUG (ScheduledTrigger-103-thread-3) [    ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_lost_trigger]
   [junit4]   2> 128466 DEBUG (AutoscalingActionExecutor-104-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"9bce9a3d82d8aTbtfp7k0wdqt33lvv72jd5a5pb",
   [junit4]   2>   "source":".auto_add_replicas",
   [junit4]   2>   "eventTime":2740986452651402,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[2740986452651402],
   [junit4]   2>     "_enqueue_time_":2741016549097102,
   [junit4]   2>     "nodeNames":["127.0.0.1:10000_solr"]}}
   [junit4]   2> 128466 DEBUG (AutoscalingActionExecutor-104-thread-1) [    ] o.a.s.c.a.ComputePlanAction -- processing event: {
   [junit4]   2>   "id":"9bce9a3d82d8aTbtfp7k0wdqt33lvv72jd5a5pb",
   [junit4]   2>   "source":".auto_add_replicas",
   [junit4]   2>   "eventTime":2740986452651402,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[2740986452651402],
   [junit4]   2>     "_enqueue_time_":2741016549097102,
   [junit4]   2>     "nodeNames":["127.0.0.1:10000_solr"]}} with context properties: {BEFORE_ACTION=[auto_add_replicas_plan]}
   [junit4]   2> 128485 INFO  (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[4CD29D80308A9012]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testNodeLostTrigger
   [junit4]   2> 128485 INFO  (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[4CD29D80308A9012]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase 
   [junit4]   2> 
   [junit4]   2> 128485 INFO  (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[4CD29D80308A9012]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 128485 INFO  (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[4CD29D80308A9012]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############
   [junit4]   2> 128485 INFO  (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[4CD29D80308A9012]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 
   [junit4]   2> 128485 INFO  (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[4CD29D80308A9012]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes:		1
   [junit4]   2> 128485 INFO  (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[4CD29D80308A9012]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes:	1
   [junit4]   2> 128485 INFO  (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[4CD29D80308A9012]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes:		1
   [junit4]   2> 128485 INFO  (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[4CD29D80308A9012]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		127.0.0.1:10000_solr
   [junit4]   2> 128486 INFO  (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[4CD29D80308A9012]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections:	[]
   [junit4]   2> 128486 INFO  (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[4CD29D80308A9012]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node:	0
   [junit4]   2> 128486 INFO  (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[4CD29D80308A9012]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node:	0
   [junit4]   2> 128486 INFO  (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[4CD29D80308A9012]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas:		0
   [junit4]   2> 128486 INFO  (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[4CD29D80308A9012]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ##########
   [junit4]   2> 128487 INFO  (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[4CD29D80308A9012]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		- autoscaling        2
   [junit4]   2> 128487 INFO  (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[4CD29D80308A9012]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		- update             3
   [junit4]   2> 128487 INFO  (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[4CD29D80308A9012]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ###########
   [junit4]   2> 128487 INFO  (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[4CD29D80308A9012]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: .auto_add_replicas
   [junit4]   2> 128488 INFO  (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[4CD29D80308A9012]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		- STARTED         1
   [junit4]   2> 128488 INFO  (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[4CD29D80308A9012]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_lost_trigger
   [junit4]   2> 128488 INFO  (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[4CD29D80308A9012]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		- STARTED         1
   [junit4]   2> 128488 INFO  (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[4CD29D80308A9012]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		- SUCCEEDED       1
   [junit4]   2> 128630 INFO  (TEST-TestTriggerIntegration.testListeners-seed#[4CD29D80308A9012]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testListeners
   [junit4]   2> 128630 DEBUG (TEST-TestTriggerIntegration.testListeners-seed#[4CD29D80308A9012]) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 5
   [junit4]   2> 128631 DEBUG (TEST-TestTriggerIntegration.testListeners-seed#[4CD29D80308A9012]) [    ] o.a.s.c.a.s.SimNodeStateProvider - removing dead node values: 127.0.0.1:10000_solr
   [junit4]   2> 128631 INFO  (TEST-TestTriggerIntegration.testListeners-seed#[4CD29D80308A9012]) [    ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
   [junit4]   2> 128631 DEBUG (TEST-TestTriggerIntegration.testListeners-seed#[4CD29D80308A9012]) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 128647 WARN  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 128703 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 128740 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 6
   [junit4]   2> 128740 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10002_solr]
   [junit4]   2> 128740 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion -1
   [junit4]   2> 128740 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 6
   [junit4]   2> 128757 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 128757 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion 6
   [junit4]   2> 128757 DEBUG (ScheduledTrigger-106-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 128775 INFO  (AutoscalingActionExecutor-104-thread-1) [    ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 2741032042 sessionWrapper.createTime 2741019264859852, this.sessionWrapper.createTime 2741019264859852 
   [junit4]   2> 128775 DEBUG (AutoscalingActionExecutor-104-thread-1) [    ] o.a.s.c.a.ExecutePlanAction -- processing event: {
   [junit4]   2>   "id":"9bce9a3d82d8aTbtfp7k0wdqt33lvv72jd5a5pb",
   [junit4]   2>   "source":".auto_add_replicas",
   [junit4]   2>   "eventTime":2740986452651402,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[2740986452651402],
   [junit4]   2>     "_enqueue_time_":2741016549097102,
   [junit4]   2>     "nodeNames":["127.0.0.1:10000_solr"]}} with context properties: {AFTER_ACTION=[auto_add_replicas_plan], BEFORE_ACTION=[auto_add_replicas_plan, execute_plan]}
   [junit4]   2> 128775 INFO  (AutoscalingActionExecutor-104-thread-1) [    ] o.a.s.c.a.ExecutePlanAction No operations to execute for event: {
   [junit4]   2>   "id":"9bce9a3d82d8aTbtfp7k0wdqt33lvv72jd5a5pb",
   [junit4]   2>   "source":".auto_add_replicas",
   [junit4]   2>   "eventTime":2740986452651402,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[2740986452651402],
   [junit4]   2>     "_enqueue_time_":2741016549097102,
   [junit4]   2>     "nodeNames":["127.0.0.1:10000_solr"]}}
   [junit4]   2> 128775 WARN  (AutoscalingActionExecutor-104-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Exception executing actions
   [junit4]   2> org.apache.lucene.store.AlreadyClosedException: ScheduledTrigger .auto_add_replicas has been closed.
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.ScheduledTriggers$TriggerWrapper.dequeue(ScheduledTriggers.java:529)
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:314)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 128775 DEBUG (AutoscalingActionExecutor-104-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing took 15489 ms for event id=9bce9a3d82d8aTbtfp7k0wdqt33lvv72jd5a5pb
   [junit4]   2> 128776 DEBUG (simCloudManagerPool-105-thread-1) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 128777 DEBUG (simCloudManagerPool-105-thread-1) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 7
   [junit4]   2> 128777 DEBUG (simCloudManagerPool-105-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10002_solr]
   [junit4]   2> 128794 DEBUG (ScheduledTrigger-106-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 128813 DEBUG (simCloudManagerPool-105-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10002_solr]
   [junit4]   2> 128813 DEBUG (simCloudManagerPool-105-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated with properties: {event=nodeAdded, waitFor=3, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}, {name=test1, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestDummyAction}], enabled=true}
   [junit4]   2> 128813 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 7
   [junit4]   2> 128813 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 128849 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 7, lastZnodeVersion 7
   [junit4]   2> 128848 DEBUG (ScheduledTrigger-106-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 128884 DEBUG (ScheduledTrigger-106-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
   [junit4]   2> 128942 DEBUG (simCloudManagerPool-105-thread-2) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 128942 DEBUG (simCloudManagerPool-105-thread-2) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 8
   [junit4]   2> 128942 DEBUG (simCloudManagerPool-105-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10002_solr]
   [junit4]   2> 128942 DEBUG (simCloudManagerPool-105-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10002_solr]
   [junit4]   2> 128942 DEBUG (simCloudManagerPool-105-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated with properties: {event=nodeAdded, waitFor=3, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}, {name=test1, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestDummyAction}], enabled=true}
   [junit4]   2> 128942 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 8
   [junit4]   2> 128943 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 8, lastZnodeVersion 8
   [junit4]   2> 128943 DEBU

[...truncated too long message...]

 org.apache.lucene.util.fst.FSTEnum.doSeekExact(FSTEnum.java:441)
   [junit4]   2>         at org.apache.lucene.util.fst.BytesRefFSTEnum.seekExact(BytesRefFSTEnum.java:84)
   [junit4]   2>         at org.apache.lucene.codecs.memory.MemoryPostingsFormat$FSTTermsEnum.seekExact(MemoryPostingsFormat.java:752)
   [junit4]   2>         at org.apache.lucene.index.MultiTermsEnum.seekExact(MultiTermsEnum.java:167)
   [junit4]   2>         at org.apache.lucene.index.CheckIndex.testTermVectors(CheckIndex.java:2469)
   [junit4]   2>         at org.apache.lucene.util.TestUtil.checkReader(TestUtil.java:339)
   [junit4]   2>         at org.apache.lucene.util.TestUtil.checkReader(TestUtil.java:319)
   [junit4]   2>         at org.apache.solr.uninverting.TestDocTermOrds.testTriggerUnInvertLimit(TestDocTermOrds.java:175)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {field=PostingsFormat(name=Memory), foo=PostingsFormat(name=Memory), id=PostingsFormat(name=Memory)}, docValues:{}, maxPointsInLeafNode=553, maxMBSortInHeap=7.1963053642827495, sim=RandomSimilarity(queryNorm=false): {field=DFR I(n)2}, locale=zh-HK, timezone=Pacific/Pago_Pago
   [junit4]   2> NOTE: Linux 4.4.0-112-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=2,free=85819224,total=535298048
   [junit4]   2> NOTE: All tests run in this JVM: [TestNodeAddedTrigger, RankQueryTest, TestIndexingPerformance, TestJsonFacets, AutoAddReplicasIntegrationTest, BigEndianAscendingWordSerializerTest, TestHighlightDedupGrouping, SpellingQueryConverterTest, TestLuceneMatchVersion, TestCustomSort, TestDelegationWithHadoopAuth, BadIndexSchemaTest, AddSchemaFieldsUpdateProcessorFactoryTest, TestManagedSchema, TestQuerySenderNoQuery, BasicDistributedZkTest, TestCollectionAPIs, TestExtendedDismaxParser, ResponseBuilderTest, DebugComponentTest, ForceLeaderTest, ShardRoutingTest, TestGraphTermsQParserPlugin, HDFSCollectionsAPITest, TestCloudSchemaless, HttpTriggerListenerTest, TestRTimerTree, TestDocumentBuilder, CustomHighlightComponentTest, OverseerCollectionConfigSetProcessorTest, ActionThrottleTest, V2StandaloneTest, HdfsCollectionsAPIDistributedZkTest, TestSSLRandomization, TestClusterStateMutator, TestExecutePlanAction, ClassificationUpdateProcessorFactoryTest, TestDocTermOrds]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestDocTermOrds -Dtests.seed=4CD29D80308A9012 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.badapples=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=zh-HK -Dtests.timezone=Pacific/Pago_Pago -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   0.00s J0 | TestDocTermOrds (suite) <<<
   [junit4]    > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([4CD29D80308A9012]:0)
   [junit4] Completed [477/780 (9!)] on J0 in 7223.16s, 7 tests, 2 errors <<< FAILURES!

[...truncated 8411 lines...]
   [junit4] Suite: org.apache.solr.ltr.TestLTRReRankingPipeline
   [junit4]   2> 113359 INFO  (TEST-TestLTRReRankingPipeline.testDifferentTopN-seed#[A35BB49AA5253521]) [    ] o.a.s.l.TestLTRReRankingPipeline rerank 1 documents 
   [junit4]   2> 113360 INFO  (TEST-TestLTRReRankingPipeline.testDifferentTopN-seed#[A35BB49AA5253521]) [    ] o.a.s.l.TestLTRReRankingPipeline doc 0 in pos 0
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLTRReRankingPipeline -Dtests.method=testDifferentTopN -Dtests.seed=A35BB49AA5253521 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.badapples=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=th -Dtests.timezone=America/Halifax -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE 0.88s J0 | TestLTRReRankingPipeline.testDifferentTopN <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: expected:<1.0> but was:<0.0>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([A35BB49AA5253521:52FAC6CA909EFFB3]:0)
   [junit4]    > 	at org.apache.solr.ltr.TestLTRReRankingPipeline.testDifferentTopN(TestLTRReRankingPipeline.java:255)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/build/contrib/solr-ltr/test/J0/temp/solr.ltr.TestLTRReRankingPipeline_A35BB49AA5253521-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {field=PostingsFormat(name=LuceneVarGapDocFreqInterval), id=PostingsFormat(name=Memory), final-score=PostingsFormat(name=Memory)}, docValues:{final-score=DocValuesFormat(name=Asserting)}, maxPointsInLeafNode=1002, maxMBSortInHeap=5.340371096113367, sim=RandomSimilarity(queryNorm=false): {field=IB SPL-D3(800.0)}, locale=th, timezone=America/Halifax
   [junit4]   2> NOTE: Linux 4.4.0-112-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=1,free=204007848,total=326107136
   [junit4]   2> NOTE: All tests run in this JVM: [TestRankingFeature, TestNoMatchSolrFeature, TestLTROnSolrCloud, TestFieldLengthFeature, TestDefaultWrapperModel, TestLTRReRankingPipeline]
   [junit4] Completed [28/37 (1!)] on J0 in 1.47s, 3 tests, 1 failure <<< FAILURES!

[...truncated 42031 lines...]

[JENKINS] Lucene-Solr-BadApples-NightlyTests-7.x - Build # 5 - Still Unstable

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-BadApples-NightlyTests-7.x/5/

3 tests failed.
FAILED:  org.apache.lucene.search.intervals.TestIntervalQuery.testOrderedNearQueryWidth3

Error Message:


Stack Trace:
java.lang.NullPointerException
	at __randomizedtesting.SeedInfo.seed([4DCC4B70CB39DDD4:A0E99C9F2519D7D5]:0)
	at org.apache.lucene.search.intervals.IntervalFilter.docID(IntervalFilter.java:38)
	at org.apache.lucene.search.intervals.IntervalScorer.docID(IntervalScorer.java:45)
	at org.apache.lucene.search.AssertingScorer.<init>(AssertingScorer.java:48)
	at org.apache.lucene.search.AssertingScorer.wrap(AssertingScorer.java:33)
	at org.apache.lucene.search.AssertingLeafCollector.setScorer(AssertingLeafCollector.java:43)
	at org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:182)
	at org.apache.lucene.search.AssertingBulkScorer.score(AssertingBulkScorer.java:79)
	at org.apache.lucene.search.AssertingBulkScorer.score(AssertingBulkScorer.java:63)
	at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:660)
	at org.apache.lucene.search.AssertingIndexSearcher.search(AssertingIndexSearcher.java:72)
	at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:462)
	at org.apache.lucene.search.QueryUtils.checkFirstSkipTo(QueryUtils.java:433)
	at org.apache.lucene.search.QueryUtils.check(QueryUtils.java:119)
	at org.apache.lucene.search.QueryUtils.check(QueryUtils.java:123)
	at org.apache.lucene.search.CheckHits.checkHits(CheckHits.java:183)
	at org.apache.lucene.search.intervals.TestIntervalQuery.checkHits(TestIntervalQuery.java:76)
	at org.apache.lucene.search.intervals.TestIntervalQuery.testOrderedNearQueryWidth3(TestIntervalQuery.java:85)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.cdcr.CdcrReplicationDistributedZkTest

Error Message:
9 threads leaked from SUITE scope at org.apache.solr.cloud.cdcr.CdcrReplicationDistributedZkTest:     1) Thread[id=9466, name=cdcr-replicator-3554-thread-2, state=WAITING, group=TGRP-CdcrReplicationDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=9502, name=cdcr-replicator-3554-thread-3, state=WAITING, group=TGRP-CdcrReplicationDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=9553, name=cdcr-replicator-3554-thread-5, state=WAITING, group=TGRP-CdcrReplicationDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=9410, name=cdcr-replicator-3554-thread-1, state=WAITING, group=TGRP-CdcrReplicationDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=9608, name=cdcr-replicator-3554-thread-8, state=WAITING, group=TGRP-CdcrReplicationDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=9596, name=cdcr-replicator-3554-thread-6, state=WAITING, group=TGRP-CdcrReplicationDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    7) Thread[id=9524, name=cdcr-replicator-3554-thread-4, state=WAITING, group=TGRP-CdcrReplicationDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    8) Thread[id=9602, name=cdcr-replicator-3554-thread-7, state=WAITING, group=TGRP-CdcrReplicationDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    9) Thread[id=9409, name=cdcr-scheduler-3553-thread-1, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 9 threads leaked from SUITE scope at org.apache.solr.cloud.cdcr.CdcrReplicationDistributedZkTest: 
   1) Thread[id=9466, name=cdcr-replicator-3554-thread-2, state=WAITING, group=TGRP-CdcrReplicationDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=9502, name=cdcr-replicator-3554-thread-3, state=WAITING, group=TGRP-CdcrReplicationDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=9553, name=cdcr-replicator-3554-thread-5, state=WAITING, group=TGRP-CdcrReplicationDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=9410, name=cdcr-replicator-3554-thread-1, state=WAITING, group=TGRP-CdcrReplicationDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=9608, name=cdcr-replicator-3554-thread-8, state=WAITING, group=TGRP-CdcrReplicationDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=9596, name=cdcr-replicator-3554-thread-6, state=WAITING, group=TGRP-CdcrReplicationDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   7) Thread[id=9524, name=cdcr-replicator-3554-thread-4, state=WAITING, group=TGRP-CdcrReplicationDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   8) Thread[id=9602, name=cdcr-replicator-3554-thread-7, state=WAITING, group=TGRP-CdcrReplicationDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   9) Thread[id=9409, name=cdcr-scheduler-3553-thread-1, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([D5C4EF253D8F25FF]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.cdcr.CdcrReplicationDistributedZkTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=9466, name=cdcr-replicator-3554-thread-2, state=WAITING, group=TGRP-CdcrReplicationDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=9502, name=cdcr-replicator-3554-thread-3, state=WAITING, group=TGRP-CdcrReplicationDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=9553, name=cdcr-replicator-3554-thread-5, state=WAITING, group=TGRP-CdcrReplicationDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=9410, name=cdcr-replicator-3554-thread-1, state=WAITING, group=TGRP-CdcrReplicationDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=9608, name=cdcr-replicator-3554-thread-8, state=WAITING, group=TGRP-CdcrReplicationDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=9596, name=cdcr-replicator-3554-thread-6, state=WAITING, group=TGRP-CdcrReplicationDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    7) Thread[id=9524, name=cdcr-replicator-3554-thread-4, state=WAITING, group=TGRP-CdcrReplicationDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    8) Thread[id=9602, name=cdcr-replicator-3554-thread-7, state=WAITING, group=TGRP-CdcrReplicationDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    9) Thread[id=9409, name=cdcr-scheduler-3553-thread-1, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=9466, name=cdcr-replicator-3554-thread-2, state=WAITING, group=TGRP-CdcrReplicationDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=9502, name=cdcr-replicator-3554-thread-3, state=WAITING, group=TGRP-CdcrReplicationDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=9553, name=cdcr-replicator-3554-thread-5, state=WAITING, group=TGRP-CdcrReplicationDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=9410, name=cdcr-replicator-3554-thread-1, state=WAITING, group=TGRP-CdcrReplicationDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=9608, name=cdcr-replicator-3554-thread-8, state=WAITING, group=TGRP-CdcrReplicationDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=9596, name=cdcr-replicator-3554-thread-6, state=WAITING, group=TGRP-CdcrReplicationDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   7) Thread[id=9524, name=cdcr-replicator-3554-thread-4, state=WAITING, group=TGRP-CdcrReplicationDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   8) Thread[id=9602, name=cdcr-replicator-3554-thread-7, state=WAITING, group=TGRP-CdcrReplicationDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   9) Thread[id=9409, name=cdcr-scheduler-3553-thread-1, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([D5C4EF253D8F25FF]:0)




Build Log:
[...truncated 1829 lines...]
   [junit4] JVM J0: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/lucene/build/core/test/temp/junit4-J0-20180404_034823_5897298924903287826980.sysout
   [junit4] >>> JVM J0 emitted unexpected output (verbatim) ----
   [junit4] codec: DummyCompressingStoredFields, pf: Lucene50, dvf: Asserting
   [junit4] <<< JVM J0: EOF ----

[...truncated 8078 lines...]
   [junit4] Suite: org.apache.lucene.search.intervals.TestIntervalQuery
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestIntervalQuery -Dtests.method=testOrderedNearQueryWidth3 -Dtests.seed=4DCC4B70CB39DDD4 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.badapples=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=sr-BA -Dtests.timezone=Antarctica/Davis -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.03s J0 | TestIntervalQuery.testOrderedNearQueryWidth3 <<<
   [junit4]    > Throwable #1: java.lang.NullPointerException
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([4DCC4B70CB39DDD4:A0E99C9F2519D7D5]:0)
   [junit4]    > 	at org.apache.lucene.search.intervals.IntervalFilter.docID(IntervalFilter.java:38)
   [junit4]    > 	at org.apache.lucene.search.intervals.IntervalScorer.docID(IntervalScorer.java:45)
   [junit4]    > 	at org.apache.lucene.search.AssertingScorer.<init>(AssertingScorer.java:48)
   [junit4]    > 	at org.apache.lucene.search.AssertingScorer.wrap(AssertingScorer.java:33)
   [junit4]    > 	at org.apache.lucene.search.AssertingLeafCollector.setScorer(AssertingLeafCollector.java:43)
   [junit4]    > 	at org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:182)
   [junit4]    > 	at org.apache.lucene.search.AssertingBulkScorer.score(AssertingBulkScorer.java:79)
   [junit4]    > 	at org.apache.lucene.search.AssertingBulkScorer.score(AssertingBulkScorer.java:63)
   [junit4]    > 	at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:660)
   [junit4]    > 	at org.apache.lucene.search.AssertingIndexSearcher.search(AssertingIndexSearcher.java:72)
   [junit4]    > 	at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:462)
   [junit4]    > 	at org.apache.lucene.search.QueryUtils.checkFirstSkipTo(QueryUtils.java:433)
   [junit4]    > 	at org.apache.lucene.search.QueryUtils.check(QueryUtils.java:119)
   [junit4]    > 	at org.apache.lucene.search.QueryUtils.check(QueryUtils.java:123)
   [junit4]    > 	at org.apache.lucene.search.CheckHits.checkHits(CheckHits.java:183)
   [junit4]    > 	at org.apache.lucene.search.intervals.TestIntervalQuery.checkHits(TestIntervalQuery.java:76)
   [junit4]    > 	at org.apache.lucene.search.intervals.TestIntervalQuery.testOrderedNearQueryWidth3(TestIntervalQuery.java:85)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {field=BlockTreeOrds(blocksize=128)}, docValues:{}, maxPointsInLeafNode=1769, maxMBSortInHeap=7.377530708325409, sim=RandomSimilarity(queryNorm=false): {field=DFI(Saturated)}, locale=sr-BA, timezone=Antarctica/Davis
   [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=1,free=189878264,total=354418688
   [junit4]   2> NOTE: All tests run in this JVM: [TestIntervals, TestIDVersionPostingsFormat, TestFloatPointNearestNeighbor, TestPayloadSpanUtil, TestCoveringQuery, TestIntervalQuery]
   [junit4] Completed [17/21 (1!)] on J0 in 0.78s, 13 tests, 1 error, 1 skipped <<< FAILURES!

[...truncated 3746 lines...]
   [junit4] Suite: org.apache.solr.cloud.cdcr.CdcrReplicationDistributedZkTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrReplicationDistributedZkTest_D5C4EF253D8F25FF-001/init-core-data-001
   [junit4]   2> 913888 INFO  (SUITE-CdcrReplicationDistributedZkTest-seed#[D5C4EF253D8F25FF]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 913890 INFO  (SUITE-CdcrReplicationDistributedZkTest-seed#[D5C4EF253D8F25FF]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 913890 INFO  (SUITE-CdcrReplicationDistributedZkTest-seed#[D5C4EF253D8F25FF]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 913890 INFO  (SUITE-CdcrReplicationDistributedZkTest-seed#[D5C4EF253D8F25FF]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
   [junit4]   2> 913891 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 913891 INFO  (Thread-1144) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 913891 INFO  (Thread-1144) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 913897 ERROR (Thread-1144) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 913991 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.c.ZkTestServer start zk server on port:55774
   [junit4]   2> 914001 INFO  (zkConnectionManagerCallback-1553-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 914004 INFO  (zkConnectionManagerCallback-1555-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 914007 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-cdcr.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 914008 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 914009 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 914010 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 914011 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 914012 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 914013 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 914014 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 914015 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 914018 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 914019 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 914020 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testTargetCollectionNotAvailable
   [junit4]   2> 914110 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T17:27:37-04:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 914111 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 914111 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 914111 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.e.j.s.session Scavenging every 660000ms
   [junit4]   2> 914112 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@66eca4ee{/,null,AVAILABLE}
   [junit4]   2> 914118 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@77a89ed0{HTTP/1.1,[http/1.1]}{127.0.0.1:38492}
   [junit4]   2> 914118 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.e.j.s.Server Started @914158ms
   [junit4]   2> 914118 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {shards=shard1, hostContext=/, hostPort=38492, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrReplicationDistributedZkTest_D5C4EF253D8F25FF-001/jetty-001/cores}
   [junit4]   2> 914119 ERROR (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 914119 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 914119 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solrâ„¢ version 7.4.0
   [junit4]   2> 914119 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 914119 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 914119 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-04-04T06:27:14.824Z
   [junit4]   2> 914121 INFO  (zkConnectionManagerCallback-1557-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 914121 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 914121 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrReplicationDistributedZkTest_D5C4EF253D8F25FF-001/jetty-001/solr.xml
   [junit4]   2> 914125 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 914125 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 914126 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@334145fd, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 914132 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:55774/solr
   [junit4]   2> 914137 INFO  (zkConnectionManagerCallback-1561-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 914138 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x1055f62a3080003, likely client has closed socket
   [junit4]   2> 914140 INFO  (zkConnectionManagerCallback-1563-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 914221 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [n:127.0.0.1:38492_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 914221 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [n:127.0.0.1:38492_    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:38492_
   [junit4]   2> 914222 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [n:127.0.0.1:38492_    ] o.a.s.c.Overseer Overseer (id=73569846168125444-127.0.0.1:38492_-n_0000000000) starting
   [junit4]   2> 914232 INFO  (zkConnectionManagerCallback-1570-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 914234 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [n:127.0.0.1:38492_    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:55774/solr ready
   [junit4]   2> 914236 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [n:127.0.0.1:38492_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38492_
   [junit4]   2> 914238 INFO  (zkCallback-1562-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 914239 INFO  (zkCallback-1569-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 914344 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [n:127.0.0.1:38492_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@334145fd
   [junit4]   2> 914352 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [n:127.0.0.1:38492_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@334145fd
   [junit4]   2> 914352 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [n:127.0.0.1:38492_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@334145fd
   [junit4]   2> 914354 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [n:127.0.0.1:38492_    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrReplicationDistributedZkTest_D5C4EF253D8F25FF-001/jetty-001/cores
   [junit4]   2> 914482 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T17:27:37-04:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 914484 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 914484 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 914484 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.e.j.s.session Scavenging every 600000ms
   [junit4]   2> 914485 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5ccd2466{/,null,AVAILABLE}
   [junit4]   2> 914485 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@5995440c{HTTP/1.1,[http/1.1]}{127.0.0.1:45810}
   [junit4]   2> 914485 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.e.j.s.Server Started @914525ms
   [junit4]   2> 914485 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {shards=shard2, hostContext=/, hostPort=45810, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrReplicationDistributedZkTest_D5C4EF253D8F25FF-001/jetty-002/cores}
   [junit4]   2> 914486 ERROR (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 914486 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 914486 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solrâ„¢ version 7.4.0
   [junit4]   2> 914486 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 914486 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 914486 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-04-04T06:27:15.191Z
   [junit4]   2> 914489 INFO  (zkConnectionManagerCallback-1572-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 914490 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 914490 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrReplicationDistributedZkTest_D5C4EF253D8F25FF-001/jetty-002/solr.xml
   [junit4]   2> 914493 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 914493 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 914494 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@334145fd, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 914499 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:55774/solr
   [junit4]   2> 914503 INFO  (zkConnectionManagerCallback-1576-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 914506 INFO  (zkConnectionManagerCallback-1578-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 914514 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [n:127.0.0.1:45810_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 914517 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [n:127.0.0.1:45810_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 914519 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [n:127.0.0.1:45810_    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 914519 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [n:127.0.0.1:45810_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45810_
   [junit4]   2> 914520 INFO  (zkCallback-1569-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 914520 INFO  (zkCallback-1562-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 914522 INFO  (zkCallback-1577-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 914784 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [n:127.0.0.1:45810_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@334145fd
   [junit4]   2> 914793 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [n:127.0.0.1:45810_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@334145fd
   [junit4]   2> 914793 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [n:127.0.0.1:45810_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@334145fd
   [junit4]   2> 914794 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [n:127.0.0.1:45810_    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrReplicationDistributedZkTest_D5C4EF253D8F25FF-001/jetty-002/cores
   [junit4]   2> 914798 INFO  (zkConnectionManagerCallback-1585-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 914799 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [n:127.0.0.1:45810_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 914799 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [n:127.0.0.1:45810_    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:55774/solr ready
   [junit4]   2> 914911 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T17:27:37-04:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 914912 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 914912 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 914912 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.e.j.s.session Scavenging every 600000ms
   [junit4]   2> 914913 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6bebd60e{/,null,AVAILABLE}
   [junit4]   2> 914913 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@30491389{HTTP/1.1,[http/1.1]}{127.0.0.1:38367}
   [junit4]   2> 914913 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.e.j.s.Server Started @914953ms
   [junit4]   2> 914913 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {shards=shard3, hostContext=/, hostPort=38367, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrReplicationDistributedZkTest_D5C4EF253D8F25FF-001/jetty-003/cores}
   [junit4]   2> 914914 ERROR (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 914914 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 914914 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solrâ„¢ version 7.4.0
   [junit4]   2> 914914 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 914914 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 914914 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-04-04T06:27:15.619Z
   [junit4]   2> 914916 INFO  (zkConnectionManagerCallback-1587-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 914917 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 914917 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrReplicationDistributedZkTest_D5C4EF253D8F25FF-001/jetty-003/solr.xml
   [junit4]   2> 914920 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 914920 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 914921 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@334145fd, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 914925 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:55774/solr
   [junit4]   2> 914930 INFO  (zkConnectionManagerCallback-1591-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 914932 INFO  (zkConnectionManagerCallback-1593-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 914938 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [n:127.0.0.1:38367_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 914939 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [n:127.0.0.1:38367_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 914941 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [n:127.0.0.1:38367_    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 914941 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [n:127.0.0.1:38367_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38367_
   [junit4]   2> 914942 INFO  (zkCallback-1569-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 914946 INFO  (zkCallback-1562-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 914946 INFO  (zkCallback-1577-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 914952 INFO  (zkCallback-1584-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 914958 INFO  (zkCallback-1592-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 915142 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [n:127.0.0.1:38367_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@334145fd
   [junit4]   2> 915151 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [n:127.0.0.1:38367_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@334145fd
   [junit4]   2> 915151 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [n:127.0.0.1:38367_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@334145fd
   [junit4]   2> 915153 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [n:127.0.0.1:38367_    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrReplicationDistributedZkTest_D5C4EF253D8F25FF-001/jetty-003/cores
   [junit4]   2> 915155 INFO  (zkConnectionManagerCallback-1600-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 915156 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [n:127.0.0.1:38367_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 915157 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [n:127.0.0.1:38367_    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:55774/solr ready
   [junit4]   2> 915285 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T17:27:37-04:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 915285 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 915285 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 915285 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.e.j.s.session Scavenging every 660000ms
   [junit4]   2> 915286 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@17f0bc33{/,null,AVAILABLE}
   [junit4]   2> 915286 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@652fa1cf{HTTP/1.1,[http/1.1]}{127.0.0.1:48634}
   [junit4]   2> 915286 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.e.j.s.Server Started @915327ms
   [junit4]   2> 915287 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {shards=shard4, hostContext=/, hostPort=48634, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrReplicationDistributedZkTest_D5C4EF253D8F25FF-001/jetty-004/cores}
   [junit4]   2> 915287 ERROR (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 915287 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 915287 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solrâ„¢ version 7.4.0
   [junit4]   2> 915287 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 915287 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 915287 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-04-04T06:27:15.992Z
   [junit4]   2> 915291 INFO  (zkConnectionManagerCallback-1602-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 915292 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 915292 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrReplicationDistributedZkTest_D5C4EF253D8F25FF-001/jetty-004/solr.xml
   [junit4]   2> 915296 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 915296 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 915298 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@334145fd, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 915306 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:55774/solr
   [junit4]   2> 915313 INFO  (zkConnectionManagerCallback-1606-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 915333 INFO  (zkConnectionManagerCallback-1608-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 915337 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [n:127.0.0.1:48634_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 915339 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [n:127.0.0.1:48634_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 915340 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [n:127.0.0.1:48634_    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 915340 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [n:127.0.0.1:48634_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:48634_
   [junit4]   2> 915341 INFO  (zkCallback-1584-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 915341 INFO  (zkCallback-1577-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 915341 INFO  (zkCallback-1569-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 915343 INFO  (zkCallback-1562-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 915346 INFO  (zkCallback-1599-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 915346 INFO  (zkCallback-1592-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 915352 INFO  (zkCallback-1607-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 915533 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [n:127.0.0.1:48634_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@334145fd
   [junit4]   2> 915542 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [n:127.0.0.1:48634_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@334145fd
   [junit4]   2> 915542 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [n:127.0.0.1:48634_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@334145fd
   [junit4]   2> 915544 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [n:127.0.0.1:48634_    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrReplicationDistributedZkTest_D5C4EF253D8F25FF-001/jetty-004/cores
   [junit4]   2> 915556 INFO  (zkConnectionManagerCallback-1615-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 915557 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [n:127.0.0.1:48634_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 915558 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [n:127.0.0.1:48634_    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:55774/solr ready
   [junit4]   2> 915601 INFO  (zkConnectionManagerCallback-1620-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 915602 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 915603 INFO  (TEST-CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable-seed#[D5C4EF253D8F25FF]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:55774/solr ready
   [junit4]   2> 915604 INFO  (qtp41332423-6130) [n:127.0.0.1:38367_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=tmp_collection&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 915607 INFO  (OverseerThreadFactory-2621-thread-1) [    ] o.a.s.c.a.c.CreateCollectionCmd Create collection tmp_collection
   [junit4]   2> 915608 WARN  (OverseerThreadFactory-2621-thread-1) [    ] o.a.s.c.a.c.CreateCollectionCmd It is unusual to create a collection (tmp_collection) without cores.
   [junit4]   2> 915723 INFO  (qtp41332423-6130) [n:127.0.0.1:38367_    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 915723 INFO  (qtp41332423-6130) [n:127.0.0.1:38367_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=tmp_collection&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&wt=javabin&version=2} status=0 QTime=119
   [junit4]   2> 915724 INFO  (qtp41332423-6132) [n:127.0.0.1:38367_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:38492_&action=ADDREPLICA&collection=tmp_collection&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 915727 INFO  (OverseerCollectionConfigSetProcessor-73569846168125444-127.0.0.1:38492_-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 915745 INFO  (OverseerThreadFactory-2621-thread-2) [    ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:38492_ for creating new replica
   [junit4]   2> 915765 INFO  (qtp1540928403-6070) [n:127.0.0.1:38492_    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=tmp_collection_shard1_replica_n41&action=CREATE&collection=tmp_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 915766 INFO  (qtp1540928403-6070) [n:127.0.0.1:38492_    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 915869 INFO  (zkCallback-1562-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tmp_collection/state.json] for collection [tmp_collection] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 916778 INFO  (qtp1540928403-6070) [n:127.0.0.1:38492_ c:tmp_collection s:shard1  x:tmp_collection_shard1_replica_n41] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.4.0
   [junit4]   2> 916787 INFO  (qtp1540928403-6070) [n:127.0.0.1:38492_ c:tmp_collection s:shard1  x:tmp_collection_shard1_replica_n41] o.a.s.s.IndexSchema [tmp_collection_shard1_replica_n41] Schema name=test
   [junit4]   2> 916861 INFO  (qtp1540928403-6070) [n:127.0.0.1:38492_ c:tmp_collection s:shard1  x:tmp_collection_shard1_replica_n41] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 916872 INFO  (qtp1540928403-6070) [n:127.0.0.1:38492_ c:tmp_collection s:shard1  x:tmp_collection_shard1_replica_n41] o.a.s.c.CoreContainer Creating SolrCore 'tmp_collection_shard1_replica_n41' using configuration from collection tmp_collection, trusted=true
   [junit4]   2> 916873 INFO  (qtp1540928403-6070) [n:127.0.0.1:38492_ c:tmp_collection s:shard1  x:tmp_collection_shard1_replica_n41] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.tmp_collection.shard1.replica_n41' (registry 'solr.core.tmp_collection.shard1.replica_n41') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@334145fd
   [junit4]   2> 916873 INFO  (qtp1540928403-6070) [n:127.0.0.1:38492_ c:tmp_collection s:shard1  x:tmp_collection_shard1_replica_n41] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 916873 INFO  (qtp1540928403-6070) [n:127.0.0.1:38492_ c:tmp_collection s:shard1  x:tmp_collection_shard1_replica_n41] o.a.s.c.SolrCore [[tmp_collection_shard1_replica_n41] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrReplicationDistributedZkTest_D5C4EF253D8F25FF-001/jetty-001/cores/tmp_collection_shard1_replica_n41], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrReplicationDistributedZkTest_D5C4EF253D8F25FF-001/jetty-001/cores/tmp_collection_shard1_replica_n41/data/]
   [junit4]   2> 916876 INFO  (qtp1540928403-6070) [n:127.0.0.1:38492_ c:tmp_collection s:shard1  x:tmp_collection_shard1_replica_n41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=10, maxMergeAtOnceExplicit=33, maxMergedSegmentMB=94.6865234375, floorSegmentMB=2.072265625, forceMergeDeletesPctAllowed=14.3777949511094, segmentsPerTier=21.0, maxCFSSegmentSizeMB=0.904296875, noCFSRatio=0.5069977244119073
   [junit4]   2> 916922 INFO  (qtp1540928403-6070) [n:127.0.0.1:38492_ c:tmp_collection s:shard1  x:tmp_collection_shard1_replica_n41] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.CdcrUpdateLog
   [junit4]   2> 916922 INFO  (qtp1540928403-6070) [n:127.0.0.1:38492_ c:tmp_collection s:shard1  x:tmp_collection_shard1_replica_n41] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 916923 INFO  (qtp1540928403-6070) [n:127.0.0.1:38492_ c:tmp_collection s:shard1  x:tmp_collection_shard1_replica_n41] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 916923 INFO  (qtp1540928403-6070) [n:127.0.0.1:38492_ c:tmp_collection s:shard1  x:tmp_collection_shard1_replica_n41] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 916924 INFO  (qtp1540928403-6070) [n:127.0.0.1:38492_ c:tmp_collection s:shard1  x:tmp_collection_shard1_replica_n41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=28, maxMergeAtOnceExplicit=21, maxMergedSegmentMB=67.7509765625, floorSegmentMB=1.63671875, forceMergeDeletesPctAllowed=14.29874184502826, segmentsPerTier=23.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 916924 INFO  (qtp1540928403-6070) [n:127.0.0.1:38492_ c:tmp_collection s:shard1  x:tmp_collection_shard1_replica_n41] o.a.s.s.SolrIndexSearcher Opening [Searcher@2fd1048a[tmp_collection_shard1_replica_n41] main]
   [junit4]   2> 916925 INFO  (qtp1540928403-6070) [n:127.0.0.1:38492_ c:tmp_collection s:shard1  x:tmp_collection_shard1_replica_n41] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 916926 INFO  (qtp1540928403-6070) [n:127.0.0.1:38492_ c:tmp_collection s:shard1  x:tmp_collection_shard1_replica_n41] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 916926 INFO  (qtp1540928403-6070) [n:127.0.0.1:38492_ c:tmp_collection s:shard1  x:tmp_collection_shard1_replica_n41] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 916929 INFO  (qtp1540928403-6070) [n:127.0.0.1:38492_ c:tmp_collection s:shard1  x:tmp_collection_shard1_replica_n41] o.a.s.h.CdcrBufferStateManager Created znode /collections/tmp_collection/cdcr/state/buffer
   [junit4]   2> 916930 INFO  (qtp1540928403-6070) [n:127.0.0.1:38492_ c:tmp_collection s:shard1  x:tmp_collection_shard1_replica_n41] o.a.s.h.CdcrProcessStateManager Created znode /collections/tmp_collection/cdcr/state/process
   [junit4]   2> 916930 INFO  (qtp1540928403-6070) [n:127.0.0.1:38492_ c:tmp_collection s:shard1  x:tmp_collection_shard1_replica_n41] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1596795899226357760
   [junit4]   2> 916935 INFO  (qtp1540928403-6070) [n:127.0.0.1:38492_ c:tmp_collection s:shard1  x:tmp_collection_shard1_replica_n41] o.a.s.c.ZkShardTerms Successful update of terms at /collections/tmp_collection/terms/shard1 to Terms{values={core_node42=0}, version=0}
   [junit4]   2> 916936 INFO  (searcherExecutor-2642-thread-1-processing-n:127.0.0.1:38492_ x:tmp_collection_shard1_replica_n41 c:tmp_collection s:shard1) [n:127.0.0.1:38492_ c:tmp_collection s:shard1  x:tmp_collection_shard1_replica_n41] o.a.s.c.SolrCore [tmp_collection_shard1_replica_n41] Registered new searcher Searcher@2fd1048a[tmp_collection_shard1_replica_n41] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 916937 INFO  (qtp1540928403-6070) [n:127.0.0.1:38492_ c:tmp_collection s:shard1  x:tmp_collection_shard1_replica_n41] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 916937 INFO  (qtp1540928403-6070) [n:127.0.0.1:38492_ c:tmp_collection s:shard1  x:tmp_collection_shard1_replica_n41] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 916937 INFO  (qtp1540928403-6070) [n:127.0.0.1:38492_ c:tmp_collection s:shard1  x:tmp_collection_shard1_replica_n41] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:38492/tmp_collection_shard1_replica_n41/
   [junit4]   2> 916937 INFO  (qtp1540928403-6070) [n:127.0.0.1:38492_ c:tmp_collection s:shard1  x:tmp_collection_shard1_replica_n41] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 916937 INFO  (qtp1540928403-6070) [n:127.0.0.1:38492_ c:tmp_collection s:shard1  x:tmp_collection_shard1_replica_n41] o.a.s.c.SyncStrategy http://127.0.0.1:38492/tmp_collection_shard1_replica_n41/ has no replicas
   [junit4]   2> 916937 INFO  (qtp1540928403-6070) [n:127.0.0.1:38492_ c:tmp_collection s:shard1  x:tmp_collection_shard1_replica_n41] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 916940 INFO  (zkCallback-1562-thread-1) [    ] o.a.s.h.CdcrLeaderStateManager Received new leader state @ tmp_collection:shard1
   [junit4]   2> 916942 INFO  (qtp1540928403-6070) [n:127.0.0.1:38492_ c:tmp_collection s:shard1  x:tmp_collection_shard1_replica_n41] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:38492/tmp_collection_shard1_replica_n41/ shard1
   [junit4]   2> 917045 INFO  (zkCallback-1562-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeD

[...truncated too long message...]

ame=cdcr-scheduler-3553-thread-1, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([D5C4EF253D8F25FF]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   [junit4]    >    1) Thread[id=9466, name=cdcr-replicator-3554-thread-2, state=WAITING, group=TGRP-CdcrReplicationDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    2) Thread[id=9502, name=cdcr-replicator-3554-thread-3, state=WAITING, group=TGRP-CdcrReplicationDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    3) Thread[id=9553, name=cdcr-replicator-3554-thread-5, state=WAITING, group=TGRP-CdcrReplicationDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    4) Thread[id=9410, name=cdcr-replicator-3554-thread-1, state=WAITING, group=TGRP-CdcrReplicationDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    5) Thread[id=9608, name=cdcr-replicator-3554-thread-8, state=WAITING, group=TGRP-CdcrReplicationDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    6) Thread[id=9596, name=cdcr-replicator-3554-thread-6, state=WAITING, group=TGRP-CdcrReplicationDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    7) Thread[id=9524, name=cdcr-replicator-3554-thread-4, state=WAITING, group=TGRP-CdcrReplicationDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    8) Thread[id=9602, name=cdcr-replicator-3554-thread-7, state=WAITING, group=TGRP-CdcrReplicationDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    9) Thread[id=9409, name=cdcr-scheduler-3553-thread-1, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([D5C4EF253D8F25FF]:0)
   [junit4] Completed [316/794 (1!)] on J2 in 1767.80s, 11 tests, 2 errors <<< FAILURES!

[...truncated 51035 lines...]

[JENKINS] Lucene-Solr-BadApples-NightlyTests-7.x - Build # 4 - Still Unstable

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-BadApples-NightlyTests-7.x/4/

7 tests failed.
FAILED:  org.apache.solr.cloud.RollingRestartTest.test

Error Message:
Unable to restart (#9): CloudJettyRunner [url=http://127.0.0.1:58765/collection1_shard1_replica_n63]

Stack Trace:
java.lang.AssertionError: Unable to restart (#9): CloudJettyRunner [url=http://127.0.0.1:58765/collection1_shard1_replica_n63]
	at __randomizedtesting.SeedInfo.seed([EEA866DCFE8E8CEC:66FC59065072E114]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.cloud.RollingRestartTest.restartWithRolesTest(RollingRestartTest.java:103)
	at org.apache.solr.cloud.RollingRestartTest.test(RollingRestartTest.java:52)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.hdfs.HdfsRestartWhileUpdatingTest.test

Error Message:
There are still nodes recoverying - waited for 320 seconds

Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 320 seconds
	at __randomizedtesting.SeedInfo.seed([EEA866DCFE8E8CEC:66FC59065072E114]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:185)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:921)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1478)
	at org.apache.solr.cloud.RestartWhileUpdatingTest.test(RestartWhileUpdatingTest.java:144)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsRestartWhileUpdatingTest

Error Message:
7 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsRestartWhileUpdatingTest:     1) Thread[id=35037, name=searcherExecutor-7737-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=35113, name=searcherExecutor-7759-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=34998, name=searcherExecutor-7726-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=35209, name=searcherExecutor-7776-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=35072, name=searcherExecutor-7748-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=35277, name=searcherExecutor-7798-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    7) Thread[id=35244, name=searcherExecutor-7787-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 7 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsRestartWhileUpdatingTest: 
   1) Thread[id=35037, name=searcherExecutor-7737-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=35113, name=searcherExecutor-7759-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=34998, name=searcherExecutor-7726-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=35209, name=searcherExecutor-7776-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=35072, name=searcherExecutor-7748-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=35277, name=searcherExecutor-7798-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   7) Thread[id=35244, name=searcherExecutor-7787-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([EEA866DCFE8E8CEC]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsRestartWhileUpdatingTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=35037, name=searcherExecutor-7737-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=35113, name=searcherExecutor-7759-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=34998, name=searcherExecutor-7726-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=35209, name=searcherExecutor-7776-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=35072, name=searcherExecutor-7748-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=35277, name=searcherExecutor-7798-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    7) Thread[id=35244, name=searcherExecutor-7787-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=35037, name=searcherExecutor-7737-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=35113, name=searcherExecutor-7759-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=34998, name=searcherExecutor-7726-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=35209, name=searcherExecutor-7776-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=35072, name=searcherExecutor-7748-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=35277, name=searcherExecutor-7798-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   7) Thread[id=35244, name=searcherExecutor-7787-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([EEA866DCFE8E8CEC]:0)


FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestStressReplication

Error Message:
expected:<983> but was:<971>

Stack Trace:
java.lang.AssertionError: expected:<983> but was:<971>
	at __randomizedtesting.SeedInfo.seed([EEA866DCFE8E8CEC:3503661AFBA6E55F]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.handler.TestReplicationHandler.doTestStressReplication(TestReplicationHandler.java:900)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestIndexFetchWithMasterUrl

Error Message:
expected:<null> but was:<.numFound:499!=495>

Stack Trace:
java.lang.AssertionError: expected:<null> but was:<.numFound:499!=495>
	at __randomizedtesting.SeedInfo.seed([EEA866DCFE8E8CEC:F51397A39F31F45A]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:147)
	at org.apache.solr.handler.TestReplicationHandler.doTestIndexFetchWithMasterUrl(TestReplicationHandler.java:800)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.update.processor.AtomicUpdateProcessorFactoryTest.testMultipleThreads

Error Message:
Exception during query

Stack Trace:
java.lang.RuntimeException: Exception during query
	at __randomizedtesting.SeedInfo.seed([EEA866DCFE8E8CEC:C29A1151724AFA68]:0)
	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:914)
	at org.apache.solr.update.processor.AtomicUpdateProcessorFactoryTest.testMultipleThreads(AtomicUpdateProcessorFactoryTest.java:264)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.RuntimeException: REQUEST FAILED: xpath=//result[@numFound=1]
	xml response was: <?xml version="1.0" encoding="UTF-8"?>
<response>
<lst name="responseHeader"><int name="status">0</int><int name="QTime">0</int></lst><result name="response" numFound="0" start="0"></result>
</response>

	request was:q=int_i:20&wt=xml
	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:907)
	... 40 more




Build Log:
[...truncated 1843 lines...]
   [junit4] JVM J0: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/lucene/build/core/test/temp/junit4-J0-20180328_004813_6234987678056290984557.sysout
   [junit4] >>> JVM J0 emitted unexpected output (verbatim) ----
   [junit4] codec: Asserting, pf: Asserting, dvf: Direct
   [junit4] <<< JVM J0: EOF ----

[...truncated 11210 lines...]
   [junit4] Suite: org.apache.solr.cloud.RollingRestartTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_EEA866DCFE8E8CEC-001/init-core-data-001
   [junit4]   2> 69385 WARN  (SUITE-RollingRestartTest-seed#[EEA866DCFE8E8CEC]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=3 numCloses=3
   [junit4]   2> 69385 INFO  (SUITE-RollingRestartTest-seed#[EEA866DCFE8E8CEC]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 69387 INFO  (SUITE-RollingRestartTest-seed#[EEA866DCFE8E8CEC]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 69387 INFO  (SUITE-RollingRestartTest-seed#[EEA866DCFE8E8CEC]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 69387 INFO  (SUITE-RollingRestartTest-seed#[EEA866DCFE8E8CEC]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
   [junit4]   2> 69395 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 69405 INFO  (Thread-74) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 69405 INFO  (Thread-74) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 69412 ERROR (Thread-74) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 69504 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.a.s.c.ZkTestServer start zk server on port:36783
   [junit4]   2> 69508 INFO  (zkConnectionManagerCallback-115-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 69514 INFO  (zkConnectionManagerCallback-117-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 69526 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 69531 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 69534 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 69536 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 69537 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 69539 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 69540 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 69542 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 69544 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 69546 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 69548 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 69550 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
   [junit4]   2> 69711 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T19:27:37-02:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 69713 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 69714 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 69714 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.e.j.s.session Scavenging every 660000ms
   [junit4]   2> 69714 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4a536095{/,null,AVAILABLE}
   [junit4]   2> 69722 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@94e9525{HTTP/1.1,[http/1.1]}{127.0.0.1:44434}
   [junit4]   2> 69723 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.e.j.s.Server Started @69778ms
   [junit4]   2> 69723 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_EEA866DCFE8E8CEC-001/tempDir-001/control/data, hostContext=/, hostPort=44434, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_EEA866DCFE8E8CEC-001/control-001/cores}
   [junit4]   2> 69723 ERROR (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 69723 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 69723 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.4.0
   [junit4]   2> 69723 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 69724 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 69724 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-03-28T03:54:47.166Z
   [junit4]   2> 69737 INFO  (zkConnectionManagerCallback-119-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 69740 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 69740 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_EEA866DCFE8E8CEC-001/control-001/solr.xml
   [junit4]   2> 69745 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 69746 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 69747 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@37623e70, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 69752 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36783/solr
   [junit4]   2> 69759 INFO  (zkConnectionManagerCallback-123-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 69774 INFO  (zkConnectionManagerCallback-125-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 69953 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [n:127.0.0.1:44434_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 69954 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [n:127.0.0.1:44434_    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:44434_
   [junit4]   2> 69955 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [n:127.0.0.1:44434_    ] o.a.s.c.Overseer Overseer (id=73529610549395460-127.0.0.1:44434_-n_0000000000) starting
   [junit4]   2> 69996 INFO  (zkConnectionManagerCallback-130-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 69999 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [n:127.0.0.1:44434_    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:36783/solr ready
   [junit4]   2> 70038 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [n:127.0.0.1:44434_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:44434_
   [junit4]   2> 70072 INFO  (zkCallback-129-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 70076 INFO  (zkCallback-124-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 70725 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [n:127.0.0.1:44434_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@37623e70
   [junit4]   2> 70737 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [n:127.0.0.1:44434_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@37623e70
   [junit4]   2> 70738 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [n:127.0.0.1:44434_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@37623e70
   [junit4]   2> 70739 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [n:127.0.0.1:44434_    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_EEA866DCFE8E8CEC-001/control-001/cores
   [junit4]   2> 70811 INFO  (zkConnectionManagerCallback-134-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 70815 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 70816 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:36783/solr ready
   [junit4]   2> 70825 INFO  (qtp523773631-500) [n:127.0.0.1:44434_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:44434_&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 70836 INFO  (OverseerThreadFactory-231-thread-1) [    ] o.a.s.c.a.c.CreateCollectionCmd Create collection control_collection
   [junit4]   2> 70952 INFO  (qtp523773631-504) [n:127.0.0.1:44434_    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 70953 INFO  (qtp523773631-504) [n:127.0.0.1:44434_    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 71061 INFO  (zkCallback-124-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 71973 INFO  (qtp523773631-504) [n:127.0.0.1:44434_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.4.0
   [junit4]   2> 72011 INFO  (qtp523773631-504) [n:127.0.0.1:44434_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
   [junit4]   2> 72181 INFO  (qtp523773631-504) [n:127.0.0.1:44434_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 72282 INFO  (qtp523773631-504) [n:127.0.0.1:44434_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
   [junit4]   2> 72283 INFO  (qtp523773631-504) [n:127.0.0.1:44434_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@37623e70
   [junit4]   2> 72303 INFO  (qtp523773631-504) [n:127.0.0.1:44434_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 72303 INFO  (qtp523773631-504) [n:127.0.0.1:44434_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_EEA866DCFE8E8CEC-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_EEA866DCFE8E8CEC-001/control-001/cores/control_collection_shard1_replica_n1/data/]
   [junit4]   2> 72315 INFO  (qtp523773631-504) [n:127.0.0.1:44434_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=5, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8475890665326732]
   [junit4]   2> 72356 WARN  (qtp523773631-504) [n:127.0.0.1:44434_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 72497 INFO  (qtp523773631-504) [n:127.0.0.1:44434_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 72497 INFO  (qtp523773631-504) [n:127.0.0.1:44434_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 72498 INFO  (qtp523773631-504) [n:127.0.0.1:44434_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 72498 INFO  (qtp523773631-504) [n:127.0.0.1:44434_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 72500 INFO  (qtp523773631-504) [n:127.0.0.1:44434_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@671ca305
   [junit4]   2> 72502 INFO  (qtp523773631-504) [n:127.0.0.1:44434_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@33aee129[control_collection_shard1_replica_n1] main]
   [junit4]   2> 72503 INFO  (qtp523773631-504) [n:127.0.0.1:44434_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 72504 INFO  (qtp523773631-504) [n:127.0.0.1:44434_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 72505 INFO  (qtp523773631-504) [n:127.0.0.1:44434_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 72505 INFO  (qtp523773631-504) [n:127.0.0.1:44434_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1596152128415465472
   [junit4]   2> 72513 INFO  (searcherExecutor-234-thread-1-processing-n:127.0.0.1:44434_ x:control_collection_shard1_replica_n1 c:control_collection s:shard1) [n:127.0.0.1:44434_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@33aee129[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 72526 INFO  (qtp523773631-504) [n:127.0.0.1:44434_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkShardTerms Successful update of terms at /collections/control_collection/terms/shard1 to Terms{values={core_node2=0}, version=0}
   [junit4]   2> 72530 INFO  (qtp523773631-504) [n:127.0.0.1:44434_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 72531 INFO  (qtp523773631-504) [n:127.0.0.1:44434_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 72531 INFO  (qtp523773631-504) [n:127.0.0.1:44434_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:44434/control_collection_shard1_replica_n1/
   [junit4]   2> 72531 INFO  (qtp523773631-504) [n:127.0.0.1:44434_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 72531 INFO  (qtp523773631-504) [n:127.0.0.1:44434_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:44434/control_collection_shard1_replica_n1/ has no replicas
   [junit4]   2> 72531 INFO  (qtp523773631-504) [n:127.0.0.1:44434_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 72537 INFO  (qtp523773631-504) [n:127.0.0.1:44434_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:44434/control_collection_shard1_replica_n1/ shard1
   [junit4]   2> 72638 INFO  (zkCallback-124-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 72688 INFO  (qtp523773631-504) [n:127.0.0.1:44434_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 72691 INFO  (qtp523773631-504) [n:127.0.0.1:44434_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1739
   [junit4]   2> 72699 INFO  (qtp523773631-500) [n:127.0.0.1:44434_    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 72805 INFO  (zkCallback-124-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 72837 INFO  (OverseerCollectionConfigSetProcessor-73529610549395460-127.0.0.1:44434_-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 73700 INFO  (qtp523773631-500) [n:127.0.0.1:44434_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:44434_&wt=javabin&version=2} status=0 QTime=2874
   [junit4]   2> 73712 INFO  (zkConnectionManagerCallback-138-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 73714 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 73715 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:36783/solr ready
   [junit4]   2> 73715 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 73717 INFO  (qtp523773631-504) [n:127.0.0.1:44434_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 73725 INFO  (OverseerThreadFactory-231-thread-2) [    ] o.a.s.c.a.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 73726 WARN  (OverseerThreadFactory-231-thread-2) [    ] o.a.s.c.a.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
   [junit4]   2> 73949 INFO  (qtp523773631-504) [n:127.0.0.1:44434_    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 73950 INFO  (qtp523773631-504) [n:127.0.0.1:44434_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2} status=0 QTime=233
   [junit4]   2> 74114 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_EEA866DCFE8E8CEC-001/shard-1-001 of type NRT
   [junit4]   2> 74116 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T19:27:37-02:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 74117 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 74117 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 74117 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.e.j.s.session Scavenging every 600000ms
   [junit4]   2> 74117 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7dc48f9c{/,null,AVAILABLE}
   [junit4]   2> 74118 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@5841499f{HTTP/1.1,[http/1.1]}{127.0.0.1:40696}
   [junit4]   2> 74118 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.e.j.s.Server Started @74173ms
   [junit4]   2> 74118 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_EEA866DCFE8E8CEC-001/tempDir-001/jetty1, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/, hostPort=40696, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_EEA866DCFE8E8CEC-001/shard-1-001/cores}
   [junit4]   2> 74119 ERROR (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 74119 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 74119 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.4.0
   [junit4]   2> 74119 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 74119 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 74119 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-03-28T03:54:51.561Z
   [junit4]   2> 74125 INFO  (zkConnectionManagerCallback-140-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 74127 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 74127 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_EEA866DCFE8E8CEC-001/shard-1-001/solr.xml
   [junit4]   2> 74135 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 74135 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 74145 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@37623e70, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 74153 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36783/solr
   [junit4]   2> 74155 INFO  (zkConnectionManagerCallback-144-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 74163 INFO  (zkConnectionManagerCallback-146-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 74173 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [n:127.0.0.1:40696_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 74176 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [n:127.0.0.1:40696_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 74179 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [n:127.0.0.1:40696_    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 74179 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [n:127.0.0.1:40696_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40696_
   [junit4]   2> 74189 INFO  (zkCallback-129-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 74189 INFO  (zkCallback-124-thread-2) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 74217 INFO  (zkCallback-137-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 74218 INFO  (zkCallback-145-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 74543 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [n:127.0.0.1:40696_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@37623e70
   [junit4]   2> 74566 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [n:127.0.0.1:40696_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@37623e70
   [junit4]   2> 74567 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [n:127.0.0.1:40696_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@37623e70
   [junit4]   2> 74572 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [n:127.0.0.1:40696_    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_EEA866DCFE8E8CEC-001/shard-1-001/cores
   [junit4]   2> 74607 INFO  (zkConnectionManagerCallback-151-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 74609 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [n:127.0.0.1:40696_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 74610 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [n:127.0.0.1:40696_    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:36783/solr ready
   [junit4]   2> 74684 INFO  (qtp1379591919-558) [n:127.0.0.1:40696_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:40696_&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 74689 INFO  (OverseerCollectionConfigSetProcessor-73529610549395460-127.0.0.1:44434_-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 74693 INFO  (OverseerThreadFactory-231-thread-3) [    ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:40696_ for creating new replica
   [junit4]   2> 74705 INFO  (qtp1379591919-557) [n:127.0.0.1:40696_    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 74817 INFO  (zkCallback-145-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 75730 INFO  (qtp1379591919-557) [n:127.0.0.1:40696_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.4.0
   [junit4]   2> 75751 INFO  (qtp1379591919-557) [n:127.0.0.1:40696_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.s.IndexSchema [collection1_shard2_replica_n41] Schema name=test
   [junit4]   2> 75941 INFO  (qtp1379591919-557) [n:127.0.0.1:40696_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 75990 INFO  (qtp1379591919-557) [n:127.0.0.1:40696_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_n41' using configuration from collection collection1, trusted=true
   [junit4]   2> 75990 INFO  (qtp1379591919-557) [n:127.0.0.1:40696_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_n41' (registry 'solr.core.collection1.shard2.replica_n41') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@37623e70
   [junit4]   2> 75991 INFO  (qtp1379591919-557) [n:127.0.0.1:40696_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 75991 INFO  (qtp1379591919-557) [n:127.0.0.1:40696_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.SolrCore [[collection1_shard2_replica_n41] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_EEA866DCFE8E8CEC-001/shard-1-001/cores/collection1_shard2_replica_n41], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_EEA866DCFE8E8CEC-001/shard-1-001/cores/collection1_shard2_replica_n41/data/]
   [junit4]   2> 75995 INFO  (qtp1379591919-557) [n:127.0.0.1:40696_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=5, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8475890665326732]
   [junit4]   2> 76028 WARN  (qtp1379591919-557) [n:127.0.0.1:40696_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 76130 INFO  (qtp1379591919-557) [n:127.0.0.1:40696_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 76130 INFO  (qtp1379591919-557) [n:127.0.0.1:40696_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 76131 INFO  (qtp1379591919-557) [n:127.0.0.1:40696_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 76131 INFO  (qtp1379591919-557) [n:127.0.0.1:40696_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 76133 INFO  (qtp1379591919-557) [n:127.0.0.1:40696_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@5ee12771
   [junit4]   2> 76136 INFO  (qtp1379591919-557) [n:127.0.0.1:40696_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.s.SolrIndexSearcher Opening [Searcher@29f50c9[collection1_shard2_replica_n41] main]
   [junit4]   2> 76138 INFO  (qtp1379591919-557) [n:127.0.0.1:40696_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 76139 INFO  (qtp1379591919-557) [n:127.0.0.1:40696_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 76139 INFO  (qtp1379591919-557) [n:127.0.0.1:40696_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 76140 INFO  (qtp1379591919-557) [n:127.0.0.1:40696_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1596152132227039232
   [junit4]   2> 76147 INFO  (qtp1379591919-557) [n:127.0.0.1:40696_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.ZkShardTerms Successful update of terms at /collections/collection1/terms/shard2 to Terms{values={core_node42=0}, version=0}
   [junit4]   2> 76151 INFO  (qtp1379591919-557) [n:127.0.0.1:40696_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 76151 INFO  (qtp1379591919-557) [n:127.0.0.1:40696_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 76151 INFO  (qtp1379591919-557) [n:127.0.0.1:40696_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:40696/collection1_shard2_replica_n41/
   [junit4]   2> 76151 INFO  (qtp1379591919-557) [n:127.0.0.1:40696_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 76151 INFO  (qtp1379591919-557) [n:127.0.0.1:40696_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.SyncStrategy http://127.0.0.1:40696/collection1_shard2_replica_n41/ has no replicas
   [junit4]   2> 76151 INFO  (qtp1379591919-557) [n:127.0.0.1:40696_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 76153 INFO  (searcherExecutor-245-thread-1-processing-n:127.0.0.1:40696_ x:collection1_shard2_replica_n41 c:collection1 s:shard2) [n:127.0.0.1:40696_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.SolrCore [collection1_shard2_replica_n41] Registered new searcher Searcher@29f50c9[collection1_shard2_replica_n41] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 76156 INFO  (qtp1379591919-557) [n:127.0.0.1:40696_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:40696/collection1_shard2_replica_n41/ shard2
   [junit4]   2> 76257 INFO  (zkCallback-145-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 76307 INFO  (qtp1379591919-557) [n:127.0.0.1:40696_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 76312 INFO  (qtp1379591919-557) [n:127.0.0.1:40696_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1607
   [junit4]   2> 76319 INFO  (qtp1379591919-558) [n:127.0.0.1:40696_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:40696_&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2} status=0 QTime=1634
   [junit4]   2> 76413 INFO  (zkCallback-145-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 76501 INFO  (TEST-RollingRestartTest.test-seed#[EEA866DCFE8E8CEC]) [    ] o.a.s.c.AbstractFull

[...truncated too long message...]

rent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    5) Thread[id=35072, name=searcherExecutor-7748-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    6) Thread[id=35277, name=searcherExecutor-7798-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    7) Thread[id=35244, name=searcherExecutor-7787-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([EEA866DCFE8E8CEC]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   [junit4]    >    1) Thread[id=35037, name=searcherExecutor-7737-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    2) Thread[id=35113, name=searcherExecutor-7759-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    3) Thread[id=34998, name=searcherExecutor-7726-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    4) Thread[id=35209, name=searcherExecutor-7776-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    5) Thread[id=35072, name=searcherExecutor-7748-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    6) Thread[id=35277, name=searcherExecutor-7798-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    7) Thread[id=35244, name=searcherExecutor-7787-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([EEA866DCFE8E8CEC]:0)
   [junit4] Completed [495/785 (4!)] on J2 in 482.58s, 1 test, 1 failure, 2 errors <<< FAILURES!

[...truncated 50411 lines...]

Re: [JENKINS] Lucene-Solr-BadApples-NightlyTests-7.x - Build # 3 - Still Unstable

Posted by Simon Willnauer <si...@gmail.com>.
I pushed a fix for the Lucene issue. it was caused by some changes on
LUCENE-8212

On Wed, Mar 21, 2018 at 2:50 AM, Apache Jenkins Server
<je...@builds.apache.org> wrote:
> Build: https://builds.apache.org/job/Lucene-Solr-BadApples-NightlyTests-7.x/3/
>
> 10 tests failed.
> FAILED:  org.apache.lucene.index.TestIndexWriterOnVMError.testUnknownError
>
> Error Message:
> MockDirectoryWrapper: cannot close: there are still 1 open files: {_0_Asserting_0.dvm=1}
>
> Stack Trace:
> java.lang.RuntimeException: MockDirectoryWrapper: cannot close: there are still 1 open files: {_0_Asserting_0.dvm=1}
>         at __randomizedtesting.SeedInfo.seed([C1B6EA2337948F2A:2AE035A33513191A]:0)
>         at org.apache.lucene.store.MockDirectoryWrapper.close(MockDirectoryWrapper.java:841)
>         at org.apache.lucene.index.TestIndexWriterOnVMError.doTest(TestIndexWriterOnVMError.java:89)
>         at org.apache.lucene.index.TestIndexWriterOnVMError.testUnknownError(TestIndexWriterOnVMError.java:258)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:498)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
>         at java.lang.Thread.run(Thread.java:748)
> Caused by: java.lang.RuntimeException: unclosed IndexOutput: _0_Asserting_0.dvm
>         at org.apache.lucene.store.MockDirectoryWrapper.addFileHandle(MockDirectoryWrapper.java:732)
>         at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:667)
>         at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44)
>         at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
>         at org.apache.lucene.codecs.lucene70.Lucene70DocValuesConsumer.<init>(Lucene70DocValuesConsumer.java:72)
>         at org.apache.lucene.codecs.lucene70.Lucene70DocValuesFormat.fieldsConsumer(Lucene70DocValuesFormat.java:127)
>         at org.apache.lucene.codecs.asserting.AssertingDocValuesFormat.fieldsConsumer(AssertingDocValuesFormat.java:55)
>         at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:211)
>         at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addSortedField(PerFieldDocValuesFormat.java:119)
>         at org.apache.lucene.index.SortedDocValuesWriter.flush(SortedDocValuesWriter.java:163)
>         at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:248)
>         at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:132)
>         at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:463)
>         at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:556)
>         at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:672)
>         at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:454)
>         at org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:103)
>         at org.apache.lucene.index.TestIndexWriterOnVMError.doTest(TestIndexWriterOnVMError.java:175)
>         ... 37 more
>
>
> FAILED:  org.apache.solr.cloud.LIROnShardRestartTest.testAllReplicasInLIR
>
> Error Message:
> No live SolrServers available to handle this request
>
> Stack Trace:
> org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request
>         at __randomizedtesting.SeedInfo.seed([8C1E58940135899E:D68662527FB5EE79]:0)
>         at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:460)
>         at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1105)
>         at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:885)
>         at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:818)
>         at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
>         at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
>         at org.apache.solr.cloud.LIROnShardRestartTest.testAllReplicasInLIR(LIROnShardRestartTest.java:160)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:498)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
>         at java.lang.Thread.run(Thread.java:748)
>
>
> FAILED:  org.apache.solr.cloud.LIROnShardRestartTest.testSeveralReplicasInLIR
>
> Error Message:
> No live SolrServers available to handle this request
>
> Stack Trace:
> org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request
>         at __randomizedtesting.SeedInfo.seed([8C1E58940135899E:A44CFA0216FC4C7]:0)
>         at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:460)
>         at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1105)
>         at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:885)
>         at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:818)
>         at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
>         at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
>         at org.apache.solr.cloud.LIROnShardRestartTest.testSeveralReplicasInLIR(LIROnShardRestartTest.java:175)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:498)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
>         at java.lang.Thread.run(Thread.java:748)
>
>
> FAILED:  org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test
>
> Error Message:
> Expected numSlices=5 numReplicas=1 but found DocCollection(solrj_collection4//collections/solrj_collection4/state.json/33)={   "pullReplicas":"0",   "replicationFactor":"1",   "shards":{     "shard1":{       "range":"80000000-b332ffff",       "state":"active",       "replicas":{"core_node3":{           "dataDir":"hdfs://lucene2-us-west.apache.org:39207/solr_hdfs_home/solrj_collection4/core_node3/data/",           "base_url":"http://127.0.0.1:40786/ml_/hn",           "node_name":"127.0.0.1:40786_ml_%2Fhn",           "type":"NRT",           "ulogDir":"hdfs://lucene2-us-west.apache.org:39207/solr_hdfs_home/solrj_collection4/core_node3/data/tlog",           "core":"solrj_collection4_shard1_replica_n1",           "shared_storage":"true",           "state":"active",           "leader":"true"}}},     "shard2":{       "range":"b3330000-e665ffff",       "state":"active",       "replicas":{"core_node5":{           "dataDir":"hdfs://lucene2-us-west.apache.org:39207/solr_hdfs_home/solrj_collection4/core_node5/data/",           "base_url":"http://127.0.0.1:38648/ml_/hn",           "node_name":"127.0.0.1:38648_ml_%2Fhn",           "type":"NRT",           "ulogDir":"hdfs://lucene2-us-west.apache.org:39207/solr_hdfs_home/solrj_collection4/core_node5/data/tlog",           "core":"solrj_collection4_shard2_replica_n2",           "shared_storage":"true",           "state":"active",           "leader":"true"}}},     "shard3":{       "range":"e6660000-1998ffff",       "state":"active",       "replicas":{"core_node7":{           "dataDir":"hdfs://lucene2-us-west.apache.org:39207/solr_hdfs_home/solrj_collection4/core_node7/data/",           "base_url":"http://127.0.0.1:38648/ml_/hn",           "node_name":"127.0.0.1:38648_ml_%2Fhn",           "type":"NRT",           "ulogDir":"hdfs://lucene2-us-west.apache.org:39207/solr_hdfs_home/solrj_collection4/core_node7/data/tlog",           "core":"solrj_collection4_shard3_replica_n4",           "shared_storage":"true",           "state":"active",           "leader":"true"}}},     "shard4":{       "range":"19990000-4ccbffff",       "state":"active",       "replicas":{"core_node9":{           "dataDir":"hdfs://lucene2-us-west.apache.org:39207/solr_hdfs_home/solrj_collection4/core_node9/data/",           "base_url":"http://127.0.0.1:37880/ml_/hn",           "node_name":"127.0.0.1:37880_ml_%2Fhn",           "type":"NRT",           "ulogDir":"hdfs://lucene2-us-west.apache.org:39207/solr_hdfs_home/solrj_collection4/core_node9/data/tlog",           "core":"solrj_collection4_shard4_replica_n6",           "shared_storage":"true",           "state":"active",           "leader":"true"}}},     "shard5":{       "range":"4ccc0000-7fffffff",       "state":"active",       "replicas":{"core_node10":{           "core":"solrj_collection4_shard5_replica_n8",           "base_url":"http://127.0.0.1:37880/ml_/hn",           "node_name":"127.0.0.1:37880_ml_%2Fhn",           "state":"down",           "type":"NRT"}}}},   "router":{     "field":"text",     "name":"compositeId"},   "maxShardsPerNode":"5",   "autoAddReplicas":"true",   "nrtReplicas":"1",   "tlogReplicas":"0"} with /live_nodes: [127.0.0.1:37880_ml_%2Fhn, 127.0.0.1:38648_ml_%2Fhn, 127.0.0.1:38840_ml_%2Fhn, 127.0.0.1:40786_ml_%2Fhn, 127.0.0.1:45388_ml_%2Fhn]
>
> Stack Trace:
> java.lang.AssertionError: Expected numSlices=5 numReplicas=1 but found DocCollection(solrj_collection4//collections/solrj_collection4/state.json/33)={
>   "pullReplicas":"0",
>   "replicationFactor":"1",
>   "shards":{
>     "shard1":{
>       "range":"80000000-b332ffff",
>       "state":"active",
>       "replicas":{"core_node3":{
>           "dataDir":"hdfs://lucene2-us-west.apache.org:39207/solr_hdfs_home/solrj_collection4/core_node3/data/",
>           "base_url":"http://127.0.0.1:40786/ml_/hn",
>           "node_name":"127.0.0.1:40786_ml_%2Fhn",
>           "type":"NRT",
>           "ulogDir":"hdfs://lucene2-us-west.apache.org:39207/solr_hdfs_home/solrj_collection4/core_node3/data/tlog",
>           "core":"solrj_collection4_shard1_replica_n1",
>           "shared_storage":"true",
>           "state":"active",
>           "leader":"true"}}},
>     "shard2":{
>       "range":"b3330000-e665ffff",
>       "state":"active",
>       "replicas":{"core_node5":{
>           "dataDir":"hdfs://lucene2-us-west.apache.org:39207/solr_hdfs_home/solrj_collection4/core_node5/data/",
>           "base_url":"http://127.0.0.1:38648/ml_/hn",
>           "node_name":"127.0.0.1:38648_ml_%2Fhn",
>           "type":"NRT",
>           "ulogDir":"hdfs://lucene2-us-west.apache.org:39207/solr_hdfs_home/solrj_collection4/core_node5/data/tlog",
>           "core":"solrj_collection4_shard2_replica_n2",
>           "shared_storage":"true",
>           "state":"active",
>           "leader":"true"}}},
>     "shard3":{
>       "range":"e6660000-1998ffff",
>       "state":"active",
>       "replicas":{"core_node7":{
>           "dataDir":"hdfs://lucene2-us-west.apache.org:39207/solr_hdfs_home/solrj_collection4/core_node7/data/",
>           "base_url":"http://127.0.0.1:38648/ml_/hn",
>           "node_name":"127.0.0.1:38648_ml_%2Fhn",
>           "type":"NRT",
>           "ulogDir":"hdfs://lucene2-us-west.apache.org:39207/solr_hdfs_home/solrj_collection4/core_node7/data/tlog",
>           "core":"solrj_collection4_shard3_replica_n4",
>           "shared_storage":"true",
>           "state":"active",
>           "leader":"true"}}},
>     "shard4":{
>       "range":"19990000-4ccbffff",
>       "state":"active",
>       "replicas":{"core_node9":{
>           "dataDir":"hdfs://lucene2-us-west.apache.org:39207/solr_hdfs_home/solrj_collection4/core_node9/data/",
>           "base_url":"http://127.0.0.1:37880/ml_/hn",
>           "node_name":"127.0.0.1:37880_ml_%2Fhn",
>           "type":"NRT",
>           "ulogDir":"hdfs://lucene2-us-west.apache.org:39207/solr_hdfs_home/solrj_collection4/core_node9/data/tlog",
>           "core":"solrj_collection4_shard4_replica_n6",
>           "shared_storage":"true",
>           "state":"active",
>           "leader":"true"}}},
>     "shard5":{
>       "range":"4ccc0000-7fffffff",
>       "state":"active",
>       "replicas":{"core_node10":{
>           "core":"solrj_collection4_shard5_replica_n8",
>           "base_url":"http://127.0.0.1:37880/ml_/hn",
>           "node_name":"127.0.0.1:37880_ml_%2Fhn",
>           "state":"down",
>           "type":"NRT"}}}},
>   "router":{
>     "field":"text",
>     "name":"compositeId"},
>   "maxShardsPerNode":"5",
>   "autoAddReplicas":"true",
>   "nrtReplicas":"1",
>   "tlogReplicas":"0"} with /live_nodes: [127.0.0.1:37880_ml_%2Fhn, 127.0.0.1:38648_ml_%2Fhn, 127.0.0.1:38840_ml_%2Fhn, 127.0.0.1:40786_ml_%2Fhn, 127.0.0.1:45388_ml_%2Fhn]
>         at __randomizedtesting.SeedInfo.seed([8C1E58940135899E:44A674EAFC9E466]:0)
>         at org.junit.Assert.fail(Assert.java:93)
>         at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.assertSliceAndReplicaCount(SharedFSAutoReplicaFailoverTest.java:415)
>         at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:274)
>         at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:145)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:498)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
>         at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
>         at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
>         at java.lang.Thread.run(Thread.java:748)
>
>
> FAILED:  org.apache.solr.cloud.autoscaling.AutoScalingHandlerTest.testConcurrentUpdates
>
> Error Message:
> Captured an uncaught exception in thread: Thread[id=20023, name=Thread-6668, state=RUNNABLE, group=TGRP-AutoScalingHandlerTest]
>
> Stack Trace:
> com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=20023, name=Thread-6668, state=RUNNABLE, group=TGRP-AutoScalingHandlerTest]
> Caused by: java.lang.AssertionError: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteExecutionException: Error from server at https://127.0.0.1:41129/solr: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
>         at __randomizedtesting.SeedInfo.seed([8C1E58940135899E]:0)
>         at org.junit.Assert.fail(Assert.java:93)
>         at org.apache.solr.cloud.autoscaling.AutoScalingHandlerTest.lambda$testConcurrentUpdates$0(AutoScalingHandlerTest.java:758)
>         at java.lang.Thread.run(Thread.java:748)
>
>
> FAILED:  org.apache.solr.cloud.cdcr.CdcrReplicationDistributedZkTest.testUpdateLogSynchronisation
>
> Error Message:
> Timeout while trying to assert update logs @ source_collection
>
> Stack Trace:
> java.lang.AssertionError: Timeout while trying to assert update logs @ source_collection
>         at __randomizedtesting.SeedInfo.seed([8C1E58940135899E:72710037C315AA8F]:0)
>         at org.apache.solr.cloud.cdcr.CdcrReplicationDistributedZkTest.testUpdateLogSynchronisation(CdcrReplicationDistributedZkTest.java:407)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:498)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
>         at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
>         at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
>         at java.lang.Thread.run(Thread.java:748)
>
>
> FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload
>
> Error Message:
> expected:<[{indexVersion=1521590816599,generation=2,filelist=[_15f.fdt, _15f.fdx, _15f.fnm, _15f.nvd, _15f.nvm, _15f.si, _15f_Lucene50_0.doc, _15f_Lucene50_0.tim, _15f_Lucene50_0.tip, _15g.fdt, _15g.fdx, _15g.fnm, _15g.nvd, _15g.nvm, _15g.si, _15g_Lucene50_0.doc, _15g_Lucene50_0.tim, _15g_Lucene50_0.tip, _15h.fdt, _15h.fdx, _15h.fnm, _15h.nvd, _15h.nvm, _15h.si, _15h_Lucene50_0.doc, _15h_Lucene50_0.tim, _15h_Lucene50_0.tip, _15i.fdt, _15i.fdx, _15i.fnm, _15i.nvd, _15i.nvm, _15i.si, _15i_Lucene50_0.doc, _15i_Lucene50_0.tim, _15i_Lucene50_0.tip, _15j.fdt, _15j.fdx, _15j.fnm, _15j.nvd, _15j.nvm, _15j.si, _15j_Lucene50_0.doc, _15j_Lucene50_0.tim, _15j_Lucene50_0.tip, _15k.fdt, _15k.fdx, _15k.fnm, _15k.nvd, _15k.nvm, _15k.si, _15k_Lucene50_0.doc, _15k_Lucene50_0.tim, _15k_Lucene50_0.tip, _15l.fdt, _15l.fdx, _15l.fnm, _15l.nvd, _15l.nvm, _15l.si, _15l_Lucene50_0.doc, _15l_Lucene50_0.tim, _15l_Lucene50_0.tip, _15m.fdt, _15m.fdx, _15m.fnm, _15m.nvd, _15m.nvm, _15m.si, _15m_Lucene50_0.doc, _15m_Lucene50_0.tim, _15m_Lucene50_0.tip, _15n.fdt, _15n.fdx, _15n.fnm, _15n.nvd, _15n.nvm, _15n.si, _15n_Lucene50_0.doc, _15n_Lucene50_0.tim, _15n_Lucene50_0.tip, _15o.fdt, _15o.fdx, _15o.fnm, _15o.nvd, _15o.nvm, _15o.si, _15o_Lucene50_0.doc, _15o_Lucene50_0.tim, _15o_Lucene50_0.tip, _15p.fdt, _15p.fdx, _15p.fnm, _15p.nvd, _15p.nvm, _15p.si, _15p_Lucene50_0.doc, _15p_Lucene50_0.tim, _15p_Lucene50_0.tip, _15q.fdt, _15q.fdx, _15q.fnm, _15q.nvd, _15q.nvm, _15q.si, _15q_Lucene50_0.doc, _15q_Lucene50_0.tim, _15q_Lucene50_0.tip, _15r.fdt, _15r.fdx, _15r.fnm, _15r.nvd, _15r.nvm, _15r.si, _15r_Lucene50_0.doc, _15r_Lucene50_0.tim, _15r_Lucene50_0.tip, _15s.fdt, _15s.fdx, _15s.fnm, _15s.nvd, _15s.nvm, _15s.si, _15s_Lucene50_0.doc, _15s_Lucene50_0.tim, _15s_Lucene50_0.tip, _15t.fdt, _15t.fdx, _15t.fnm, _15t.nvd, _15t.nvm, _15t.si, _15t_Lucene50_0.doc, _15t_Lucene50_0.tim, _15t_Lucene50_0.tip, _15u.fdt, _15u.fdx, _15u.fnm, _15u.nvd, _15u.nvm, _15u.si, _15u_Lucene50_0.doc, _15u_Lucene50_0.tim, _15u_Lucene50_0.tip, _15v.fdt, _15v.fdx, _15v.fnm, _15v.nvd, _15v.nvm, _15v.si, _15v_Lucene50_0.doc, _15v_Lucene50_0.tim, _15v_Lucene50_0.tip, _15w.fdt, _15w.fdx, _15w.fnm, _15w.nvd, _15w.nvm, _15w.si, _15w_Lucene50_0.doc, _15w_Lucene50_0.tim, _15w_Lucene50_0.tip, _15x.fdt, _15x.fdx, _15x.fnm, _15x.nvd, _15x.nvm, _15x.si, _15x_Lucene50_0.doc, _15x_Lucene50_0.tim, _15x_Lucene50_0.tip, _15y.fdt, _15y.fdx, _15y.fnm, _15y.nvd, _15y.nvm, _15y.si, _15y_Lucene50_0.doc, _15y_Lucene50_0.tim, _15y_Lucene50_0.tip, _15z.fdt, _15z.fdx, _15z.fnm, _15z.nvd, _15z.nvm, _15z.si, _15z_Lucene50_0.doc, _15z_Lucene50_0.tim, _15z_Lucene50_0.tip, _161.fdt, _161.fdx, _161.fnm, _161.nvd, _161.nvm, _161.si, _161_Lucene50_0.doc, _161_Lucene50_0.tim, _161_Lucene50_0.tip, _162.fdt, _162.fdx, _162.fnm, _162.nvd, _162.nvm, _162.si, _162_Lucene50_0.doc, _162_Lucene50_0.tim, _162_Lucene50_0.tip, _163.fdt, _163.fdx, _163.fnm, _163.nvd, _163.nvm, _163.si, _163_Lucene50_0.doc, _163_Lucene50_0.tim, _163_Lucene50_0.tip, _164.fdt, _164.fdx, _164.fnm, _164.nvd, _164.nvm, _164.si, _164_Lucene50_0.doc, _164_Lucene50_0.tim, _164_Lucene50_0.tip, _165.fdt, _165.fdx, _165.fnm, _165.nvd, _165.nvm, _165.si, _165_Lucene50_0.doc, _165_Lucene50_0.tim, _165_Lucene50_0.tip, _166.fdt, _166.fdx, _166.fnm, _166.nvd, _166.nvm, _166.si, _166_Lucene50_0.doc, _166_Lucene50_0.tim, _166_Lucene50_0.tip, _167.fdt, _167.fdx, _167.fnm, _167.nvd, _167.nvm, _167.si, _167_Lucene50_0.doc, _167_Lucene50_0.tim, _167_Lucene50_0.tip, _168.fdt, _168.fdx, _168.fnm, _168.nvd, _168.nvm, _168.si, _168_Lucene50_0.doc, _168_Lucene50_0.tim, _168_Lucene50_0.tip, _169.fdt, _169.fdx, _169.fnm, _169.nvd, _169.nvm, _169.si, _169_Lucene50_0.doc, _169_Lucene50_0.tim, _169_Lucene50_0.tip, _16a.fdt, _16a.fdx, _16a.fnm, _16a.nvd, _16a.nvm, _16a.si, _16a_Lucene50_0.doc, _16a_Lucene50_0.tim, _16a_Lucene50_0.tip, _16b.fdt, _16b.fdx, _16b.fnm, _16b.nvd, _16b.nvm, _16b.si, _16b_Lucene50_0.doc, _16b_Lucene50_0.tim, _16b_Lucene50_0.tip, _16c.fdt, _16c.fdx, _16c.fnm, _16c.nvd, _16c.nvm, _16c.si, _16c_Lucene50_0.doc, _16c_Lucene50_0.tim, _16c_Lucene50_0.tip, _16d.fdt, _16d.fdx, _16d.fnm, _16d.nvd, _16d.nvm, _16d.si, _16d_Lucene50_0.doc, _16d_Lucene50_0.tim, _16d_Lucene50_0.tip, _16e.fdt, _16e.fdx, _16e.fnm, _16e.nvd, _16e.nvm, _16e.si, _16e_Lucene50_0.doc, _16e_Lucene50_0.tim, _16e_Lucene50_0.tip, _16f.fdt, _16f.fdx, _16f.fnm, _16f.nvd, _16f.nvm, _16f.si, _16f_Lucene50_0.doc, _16f_Lucene50_0.tim, _16f_Lucene50_0.tip, _16g.fdt, _16g.fdx, _16g.fnm, _16g.nvd, _16g.nvm, _16g.si, _16g_Lucene50_0.doc, _16g_Lucene50_0.tim, _16g_Lucene50_0.tip, _16h.fdt, _16h.fdx, _16h.fnm, _16h.nvd, _16h.nvm, _16h.si, _16h_Lucene50_0.doc, _16h_Lucene50_0.tim, _16h_Lucene50_0.tip, _16i.fdt, _16i.fdx, _16i.fnm, _16i.nvd, _16i.nvm, _16i.si, _16i_Lucene50_0.doc, _16i_Lucene50_0.tim, _16i_Lucene50_0.tip, _16j.fdt, _16j.fdx, _16j.fnm, _16j.nvd, _16j.nvm, _16j.si, _16j_Lucene50_0.doc, _16j_Lucene50_0.tim, _16j_Lucene50_0.tip, segments_2]}]> but was:<[{indexVersion=1521590816599,generation=2,filelist=[_15f.fdt, _15f.fdx, _15f.fnm, _15f.nvd, _15f.nvm, _15f.si, _15f_Lucene50_0.doc, _15f_Lucene50_0.tim, _15f_Lucene50_0.tip, _15g.fdt, _15g.fdx, _15g.fnm, _15g.nvd, _15g.nvm, _15g.si, _15g_Lucene50_0.doc, _15g_Lucene50_0.tim, _15g_Lucene50_0.tip, _15h.fdt, _15h.fdx, _15h.fnm, _15h.nvd, _15h.nvm, _15h.si, _15h_Lucene50_0.doc, _15h_Lucene50_0.tim, _15h_Lucene50_0.tip, _15i.fdt, _15i.fdx, _15i.fnm, _15i.nvd, _15i.nvm, _15i.si, _15i_Lucene50_0.doc, _15i_Lucene50_0.tim, _15i_Lucene50_0.tip, _15j.fdt, _15j.fdx, _15j.fnm, _15j.nvd, _15j.nvm, _15j.si, _15j_Lucene50_0.doc, _15j_Lucene50_0.tim, _15j_Lucene50_0.tip, _15k.fdt, _15k.fdx, _15k.fnm, _15k.nvd, _15k.nvm, _15k.si, _15k_Lucene50_0.doc, _15k_Lucene50_0.tim, _15k_Lucene50_0.tip, _15l.fdt, _15l.fdx, _15l.fnm, _15l.nvd, _15l.nvm, _15l.si, _15l_Lucene50_0.doc, _15l_Lucene50_0.tim, _15l_Lucene50_0.tip, _15m.fdt, _15m.fdx, _15m.fnm, _15m.nvd, _15m.nvm, _15m.si, _15m_Lucene50_0.doc, _15m_Lucene50_0.tim, _15m_Lucene50_0.tip, _15n.fdt, _15n.fdx, _15n.fnm, _15n.nvd, _15n.nvm, _15n.si, _15n_Lucene50_0.doc, _15n_Lucene50_0.tim, _15n_Lucene50_0.tip, _15o.fdt, _15o.fdx, _15o.fnm, _15o.nvd, _15o.nvm, _15o.si, _15o_Lucene50_0.doc, _15o_Lucene50_0.tim, _15o_Lucene50_0.tip, _15p.fdt, _15p.fdx, _15p.fnm, _15p.nvd, _15p.nvm, _15p.si, _15p_Lucene50_0.doc, _15p_Lucene50_0.tim, _15p_Lucene50_0.tip, _15q.fdt, _15q.fdx, _15q.fnm, _15q.nvd, _15q.nvm, _15q.si, _15q_Lucene50_0.doc, _15q_Lucene50_0.tim, _15q_Lucene50_0.tip, _15r.fdt, _15r.fdx, _15r.fnm, _15r.nvd, _15r.nvm, _15r.si, _15r_Lucene50_0.doc, _15r_Lucene50_0.tim, _15r_Lucene50_0.tip, _15s.fdt, _15s.fdx, _15s.fnm, _15s.nvd, _15s.nvm, _15s.si, _15s_Lucene50_0.doc, _15s_Lucene50_0.tim, _15s_Lucene50_0.tip, _15t.fdt, _15t.fdx, _15t.fnm, _15t.nvd, _15t.nvm, _15t.si, _15t_Lucene50_0.doc, _15t_Lucene50_0.tim, _15t_Lucene50_0.tip, _15u.fdt, _15u.fdx, _15u.fnm, _15u.nvd, _15u.nvm, _15u.si, _15u_Lucene50_0.doc, _15u_Lucene50_0.tim, _15u_Lucene50_0.tip, _15v.fdt, _15v.fdx, _15v.fnm, _15v.nvd, _15v.nvm, _15v.si, _15v_Lucene50_0.doc, _15v_Lucene50_0.tim, _15v_Lucene50_0.tip, _15w.fdt, _15w.fdx, _15w.fnm, _15w.nvd, _15w.nvm, _15w.si, _15w_Lucene50_0.doc, _15w_Lucene50_0.tim, _15w_Lucene50_0.tip, _15x.fdt, _15x.fdx, _15x.fnm, _15x.nvd, _15x.nvm, _15x.si, _15x_Lucene50_0.doc, _15x_Lucene50_0.tim, _15x_Lucene50_0.tip, _15y.fdt, _15y.fdx, _15y.fnm, _15y.nvd, _15y.nvm, _15y.si, _15y_Lucene50_0.doc, _15y_Lucene50_0.tim, _15y_Lucene50_0.tip, _15z.fdt, _15z.fdx, _15z.fnm, _15z.nvd, _15z.nvm, _15z.si, _15z_Lucene50_0.doc, _15z_Lucene50_0.tim, _15z_Lucene50_0.tip, _161.fdt, _161.fdx, _161.fnm, _161.nvd, _161.nvm, _161.si, _161_Lucene50_0.doc, _161_Lucene50_0.tim, _161_Lucene50_0.tip, _162.fdt, _162.fdx, _162.fnm, _162.nvd, _162.nvm, _162.si, _162_Lucene50_0.doc, _162_Lucene50_0.tim, _162_Lucene50_0.tip, _163.fdt, _163.fdx, _163.fnm, _163.nvd, _163.nvm, _163.si, _163_Lucene50_0.doc, _163_Lucene50_0.tim, _163_Lucene50_0.tip, _164.fdt, _164.fdx, _164.fnm, _164.nvd, _164.nvm, _164.si, _164_Lucene50_0.doc, _164_Lucene50_0.tim, _164_Lucene50_0.tip, _165.fdt, _165.fdx, _165.fnm, _165.nvd, _165.nvm, _165.si, _165_Lucene50_0.doc, _165_Lucene50_0.tim, _165_Lucene50_0.tip, _166.fdt, _166.fdx, _166.fnm, _166.nvd, _166.nvm, _166.si, _166_Lucene50_0.doc, _166_Lucene50_0.tim, _166_Lucene50_0.tip, _167.fdt, _167.fdx, _167.fnm, _167.nvd, _167.nvm, _167.si, _167_Lucene50_0.doc, _167_Lucene50_0.tim, _167_Lucene50_0.tip, _168.fdt, _168.fdx, _168.fnm, _168.nvd, _168.nvm, _168.si, _168_Lucene50_0.doc, _168_Lucene50_0.tim, _168_Lucene50_0.tip, _169.fdt, _169.fdx, _169.fnm, _169.nvd, _169.nvm, _169.si, _169_Lucene50_0.doc, _169_Lucene50_0.tim, _169_Lucene50_0.tip, _16a.fdt, _16a.fdx, _16a.fnm, _16a.nvd, _16a.nvm, _16a.si, _16a_Lucene50_0.doc, _16a_Lucene50_0.tim, _16a_Lucene50_0.tip, _16b.fdt, _16b.fdx, _16b.fnm, _16b.nvd, _16b.nvm, _16b.si, _16b_Lucene50_0.doc, _16b_Lucene50_0.tim, _16b_Lucene50_0.tip, _16c.fdt, _16c.fdx, _16c.fnm, _16c.nvd, _16c.nvm, _16c.si, _16c_Lucene50_0.doc, _16c_Lucene50_0.tim, _16c_Lucene50_0.tip, _16d.fdt, _16d.fdx, _16d.fnm, _16d.nvd, _16d.nvm, _16d.si, _16d_Lucene50_0.doc, _16d_Lucene50_0.tim, _16d_Lucene50_0.tip, _16e.fdt, _16e.fdx, _16e.fnm, _16e.nvd, _16e.nvm, _16e.si, _16e_Lucene50_0.doc, _16e_Lucene50_0.tim, _16e_Lucene50_0.tip, _16f.fdt, _16f.fdx, _16f.fnm, _16f.nvd, _16f.nvm, _16f.si, _16f_Lucene50_0.doc, _16f_Lucene50_0.tim, _16f_Lucene50_0.tip, _16g.fdt, _16g.fdx, _16g.fnm, _16g.nvd, _16g.nvm, _16g.si, _16g_Lucene50_0.doc, _16g_Lucene50_0.tim, _16g_Lucene50_0.tip, _16h.fdt, _16h.fdx, _16h.fnm, _16h.nvd, _16h.nvm, _16h.si, _16h_Lucene50_0.doc, _16h_Lucene50_0.tim, _16h_Lucene50_0.tip, _16i.fdt, _16i.fdx, _16i.fnm, _16i.nvd, _16i.nvm, _16i.si, _16i_Lucene50_0.doc, _16i_Lucene50_0.tim, _16i_Lucene50_0.tip, _16j.fdt, _16j.fdx, _16j.fnm, _16j.nvd, _16j.nvm, _16j.si, _16j_Lucene50_0.doc, _16j_Lucene50_0.tim, _16j_Lucene50_0.tip, segments_2]}, {indexVersion=1521590816599,generation=3,filelist=[_160.fdt, _160.fdx, _160.fnm, _160.nvd, _160.nvm, _160.si, _160_Lucene50_0.doc, _160_Lucene50_0.tim, _160_Lucene50_0.tip, _161.fdt, _161.fdx, _161.fnm, _161.nvd, _161.nvm, _161.si, _161_Lucene50_0.doc, _161_Lucene50_0.tim, _161_Lucene50_0.tip, _162.fdt, _162.fdx, _162.fnm, _162.nvd, _162.nvm, _162.si, _162_Lucene50_0.doc, _162_Lucene50_0.tim, _162_Lucene50_0.tip, _163.fdt, _163.fdx, _163.fnm, _163.nvd, _163.nvm, _163.si, _163_Lucene50_0.doc, _163_Lucene50_0.tim, _163_Lucene50_0.tip, _164.fdt, _164.fdx, _164.fnm, _164.nvd, _164.nvm, _164.si, _164_Lucene50_0.doc, _164_Lucene50_0.tim, _164_Lucene50_0.tip, _165.fdt, _165.fdx, _165.fnm, _165.nvd, _165.nvm, _165.si, _165_Lucene50_0.doc, _165_Lucene50_0.tim, _165_Lucene50_0.tip, _166.fdt, _166.fdx, _166.fnm, _166.nvd, _166.nvm, _166.si, _166_Lucene50_0.doc, _166_Lucene50_0.tim, _166_Lucene50_0.tip, _167.fdt, _167.fdx, _167.fnm, _167.nvd, _167.nvm, _167.si, _167_Lucene50_0.doc, _167_Lucene50_0.tim, _167_Lucene50_0.tip, _168.fdt, _168.fdx, _168.fnm, _168.nvd, _168.nvm, _168.si, _168_Lucene50_0.doc, _168_Lucene50_0.tim, _168_Lucene50_0.tip, _169.fdt, _169.fdx, _169.fnm, _169.nvd, _169.nvm, _169.si, _169_Lucene50_0.doc, _169_Lucene50_0.tim, _169_Lucene50_0.tip, _16a.fdt, _16a.fdx, _16a.fnm, _16a.nvd, _16a.nvm, _16a.si, _16a_Lucene50_0.doc, _16a_Lucene50_0.tim, _16a_Lucene50_0.tip, _16b.fdt, _16b.fdx, _16b.fnm, _16b.nvd, _16b.nvm, _16b.si, _16b_Lucene50_0.doc, _16b_Lucene50_0.tim, _16b_Lucene50_0.tip, _16c.fdt, _16c.fdx, _16c.fnm, _16c.nvd, _16c.nvm, _16c.si, _16c_Lucene50_0.doc, _16c_Lucene50_0.tim, _16c_Lucene50_0.tip, _16d.fdt, _16d.fdx, _16d.fnm, _16d.nvd, _16d.nvm, _16d.si, _16d_Lucene50_0.doc, _16d_Lucene50_0.tim, _16d_Lucene50_0.tip, _16e.fdt, _16e.fdx, _16e.fnm, _16e.nvd, _16e.nvm, _16e.si, _16e_Lucene50_0.doc, _16e_Lucene50_0.tim, _16e_Lucene50_0.tip, _16f.fdt, _16f.fdx, _16f.fnm, _16f.nvd, _16f.nvm, _16f.si, _16f_Lucene50_0.doc, _16f_Lucene50_0.tim, _16f_Lucene50_0.tip, _16g.fdt, _16g.fdx, _16g.fnm, _16g.nvd, _16g.nvm, _16g.si, _16g_Lucene50_0.doc, _16g_Lucene50_0.tim, _16g_Lucene50_0.tip, _16h.fdt, _16h.fdx, _16h.fnm, _16h.nvd, _16h.nvm, _16h.si, _16h_Lucene50_0.doc, _16h_Lucene50_0.tim, _16h_Lucene50_0.tip, _16i.fdt, _16i.fdx, _16i.fnm, _16i.nvd, _16i.nvm, _16i.si, _16i_Lucene50_0.doc, _16i_Lucene50_0.tim, _16i_Lucene50_0.tip, _16j.fdt, _16j.fdx, _16j.fnm, _16j.nvd, _16j.nvm, _16j.si, _16j_Lucene50_0.doc, _16j_Lucene50_0.tim, _16j_Lucene50_0.tip, segments_3]}]>
>
> Stack Trace:
> java.lang.AssertionError: expected:<[{indexVersion=1521590816599,generation=2,filelist=[_15f.fdt, _15f.fdx, _15f.fnm, _15f.nvd, _15f.nvm, _15f.si, _15f_Lucene50_0.doc, _15f_Lucene50_0.tim, _15f_Lucene50_0.tip, _15g.fdt, _15g.fdx, _15g.fnm, _15g.nvd, _15g.nvm, _15g.si, _15g_Lucene50_0.doc, _15g_Lucene50_0.tim, _15g_Lucene50_0.tip, _15h.fdt, _15h.fdx, _15h.fnm, _15h.nvd, _15h.nvm, _15h.si, _15h_Lucene50_0.doc, _15h_Lucene50_0.tim, _15h_Lucene50_0.tip, _15i.fdt, _15i.fdx, _15i.fnm, _15i.nvd, _15i.nvm, _15i.si, _15i_Lucene50_0.doc, _15i_Lucene50_0.tim, _15i_Lucene50_0.tip, _15j.fdt, _15j.fdx, _15j.fnm, _15j.nvd, _15j.nvm, _15j.si, _15j_Lucene50_0.doc, _15j_Lucene50_0.tim, _15j_Lucene50_0.tip, _15k.fdt, _15k.fdx, _15k.fnm, _15k.nvd, _15k.nvm, _15k.si, _15k_Lucene50_0.doc, _15k_Lucene50_0.tim, _15k_Lucene50_0.tip, _15l.fdt, _15l.fdx, _15l.fnm, _15l.nvd, _15l.nvm, _15l.si, _15l_Lucene50_0.doc, _15l_Lucene50_0.tim, _15l_Lucene50_0.tip, _15m.fdt, _15m.fdx, _15m.fnm, _15m.nvd, _15m.nvm, _15m.si, _15m_Lucene50_0.doc, _15m_Lucene50_0.tim, _15m_Lucene50_0.tip, _15n.fdt, _15n.fdx, _15n.fnm, _15n.nvd, _15n.nvm, _15n.si, _15n_Lucene50_0.doc, _15n_Lucene50_0.tim, _15n_Lucene50_0.tip, _15o.fdt, _15o.fdx, _15o.fnm, _15o.nvd, _15o.nvm, _15o.si, _15o_Lucene50_0.doc, _15o_Lucene50_0.tim, _15o_Lucene50_0.tip, _15p.fdt, _15p.fdx, _15p.fnm, _15p.nvd, _15p.nvm, _15p.si, _15p_Lucene50_0.doc, _15p_Lucene50_0.tim, _15p_Lucene50_0.tip, _15q.fdt, _15q.fdx, _15q.fnm, _15q.nvd, _15q.nvm, _15q.si, _15q_Lucene50_0.doc, _15q_Lucene50_0.tim, _15q_Lucene50_0.tip, _15r.fdt, _15r.fdx, _15r.fnm, _15r.nvd, _15r.nvm, _15r.si, _15r_Lucene50_0.doc, _15r_Lucene50_0.tim, _15r_Lucene50_0.tip, _15s.fdt, _15s.fdx, _15s.fnm, _15s.nvd, _15s.nvm, _15s.si, _15s_Lucene50_0.doc, _15s_Lucene50_0.tim, _15s_Lucene50_0.tip, _15t.fdt, _15t.fdx, _15t.fnm, _15t.nvd, _15t.nvm, _15t.si, _15t_Lucene50_0.doc, _15t_Lucene50_0.tim, _15t_Lucene50_0.tip, _15u.fdt, _15u.fdx, _15u.fnm, _15u.nvd, _15u.nvm, _15u.si, _15u_Lucene50_0.doc, _15u_Lucene50_0.tim, _15u_Lucene50_0.tip, _15v.fdt, _15v.fdx, _15v.fnm, _15v.nvd, _15v.nvm, _15v.si, _15v_Lucene50_0.doc, _15v_Lucene50_0.tim, _15v_Lucene50_0.tip, _15w.fdt, _15w.fdx, _15w.fnm, _15w.nvd, _15w.nvm, _15w.si, _15w_Lucene50_0.doc, _15w_Lucene50_0.tim, _15w_Lucene50_0.tip, _15x.fdt, _15x.fdx, _15x.fnm, _15x.nvd, _15x.nvm, _15x.si, _15x_Lucene50_0.doc, _15x_Lucene50_0.tim, _15x_Lucene50_0.tip, _15y.fdt, _15y.fdx, _15y.fnm, _15y.nvd, _15y.nvm, _15y.si, _15y_Lucene50_0.doc, _15y_Lucene50_0.tim, _15y_Lucene50_0.tip, _15z.fdt, _15z.fdx, _15z.fnm, _15z.nvd, _15z.nvm, _15z.si, _15z_Lucene50_0.doc, _15z_Lucene50_0.tim, _15z_Lucene50_0.tip, _161.fdt, _161.fdx, _161.fnm, _161.nvd, _161.nvm, _161.si, _161_Lucene50_0.doc, _161_Lucene50_0.tim, _161_Lucene50_0.tip, _162.fdt, _162.fdx, _162.fnm, _162.nvd, _162.nvm, _162.si, _162_Lucene50_0.doc, _162_Lucene50_0.tim, _162_Lucene50_0.tip, _163.fdt, _163.fdx, _163.fnm, _163.nvd, _163.nvm, _163.si, _163_Lucene50_0.doc, _163_Lucene50_0.tim, _163_Lucene50_0.tip, _164.fdt, _164.fdx, _164.fnm, _164.nvd, _164.nvm, _164.si, _164_Lucene50_0.doc, _164_Lucene50_0.tim, _164_Lucene50_0.tip, _165.fdt, _165.fdx, _165.fnm, _165.nvd, _165.nvm, _165.si, _165_Lucene50_0.doc, _165_Lucene50_0.tim, _165_Lucene50_0.tip, _166.fdt, _166.fdx, _166.fnm, _166.nvd, _166.nvm, _166.si, _166_Lucene50_0.doc, _166_Lucene50_0.tim, _166_Lucene50_0.tip, _167.fdt, _167.fdx, _167.fnm, _167.nvd, _167.nvm, _167.si, _167_Lucene50_0.doc, _167_Lucene50_0.tim, _167_Lucene50_0.tip, _168.fdt, _168.fdx, _168.fnm, _168.nvd, _168.nvm, _168.si, _168_Lucene50_0.doc, _168_Lucene50_0.tim, _168_Lucene50_0.tip, _169.fdt, _169.fdx, _169.fnm, _169.nvd, _169.nvm, _169.si, _169_Lucene50_0.doc, _169_Lucene50_0.tim, _169_Lucene50_0.tip, _16a.fdt, _16a.fdx, _16a.fnm, _16a.nvd, _16a.nvm, _16a.si, _16a_Lucene50_0.doc, _16a_Lucene50_0.tim, _16a_Lucene50_0.tip, _16b.fdt, _16b.fdx, _16b.fnm, _16b.nvd, _16b.nvm, _16b.si, _16b_Lucene50_0.doc, _16b_Lucene50_0.tim, _16b_Lucene50_0.tip, _16c.fdt, _16c.fdx, _16c.fnm, _16c.nvd, _16c.nvm, _16c.si, _16c_Lucene50_0.doc, _16c_Lucene50_0.tim, _16c_Lucene50_0.tip, _16d.fdt, _16d.fdx, _16d.fnm, _16d.nvd, _16d.nvm, _16d.si, _16d_Lucene50_0.doc, _16d_Lucene50_0.tim, _16d_Lucene50_0.tip, _16e.fdt, _16e.fdx, _16e.fnm, _16e.nvd, _16e.nvm, _16e.si, _16e_Lucene50_0.doc, _16e_Lucene50_0.tim, _16e_Lucene50_0.tip, _16f.fdt, _16f.fdx, _16f.fnm, _16f.nvd, _16f.nvm, _16f.si, _16f_Lucene50_0.doc, _16f_Lucene50_0.tim, _16f_Lucene50_0.tip, _16g.fdt, _16g.fdx, _16g.fnm, _16g.nvd, _16g.nvm, _16g.si, _16g_Lucene50_0.doc, _16g_Lucene50_0.tim, _16g_Lucene50_0.tip, _16h.fdt, _16h.fdx, _16h.fnm, _16h.nvd, _16h.nvm, _16h.si, _16h_Lucene50_0.doc, _16h_Lucene50_0.tim, _16h_Lucene50_0.tip, _16i.fdt, _16i.fdx, _16i.fnm, _16i.nvd, _16i.nvm, _16i.si, _16i_Lucene50_0.doc, _16i_Lucene50_0.tim, _16i_Lucene50_0.tip, _16j.fdt, _16j.fdx, _16j.fnm, _16j.nvd, _16j.nvm, _16j.si, _16j_Lucene50_0.doc, _16j_Lucene50_0.tim, _16j_Lucene50_0.tip, segments_2]}]> but was:<[{indexVersion=1521590816599,generation=2,filelist=[_15f.fdt, _15f.fdx, _15f.fnm, _15f.nvd, _15f.nvm, _15f.si, _15f_Lucene50_0.doc, _15f_Lucene50_0.tim, _15f_Lucene50_0.tip, _15g.fdt, _15g.fdx, _15g.fnm, _15g.nvd, _15g.nvm, _15g.si, _15g_Lucene50_0.doc, _15g_Lucene50_0.tim, _15g_Lucene50_0.tip, _15h.fdt, _15h.fdx, _15h.fnm, _15h.nvd, _15h.nvm, _15h.si, _15h_Lucene50_0.doc, _15h_Lucene50_0.tim, _15h_Lucene50_0.tip, _15i.fdt, _15i.fdx, _15i.fnm, _15i.nvd, _15i.nvm, _15i.si, _15i_Lucene50_0.doc, _15i_Lucene50_0.tim, _15i_Lucene50_0.tip, _15j.fdt, _15j.fdx, _15j.fnm, _15j.nvd, _15j.nvm, _15j.si, _15j_Lucene50_0.doc, _15j_Lucene50_0.tim, _15j_Lucene50_0.tip, _15k.fdt, _15k.fdx, _15k.fnm, _15k.nvd, _15k.nvm, _15k.si, _15k_Lucene50_0.doc, _15k_Lucene50_0.tim, _15k_Lucene50_0.tip, _15l.fdt, _15l.fdx, _15l.fnm, _15l.nvd, _15l.nvm, _15l.si, _15l_Lucene50_0.doc, _15l_Lucene50_0.tim, _15l_Lucene50_0.tip, _15m.fdt, _15m.fdx, _15m.fnm, _15m.nvd, _15m.nvm, _15m.si, _15m_Lucene50_0.doc, _15m_Lucene50_0.tim, _15m_Lucene50_0.tip, _15n.fdt, _15n.fdx, _15n.fnm, _15n.nvd, _15n.nvm, _15n.si, _15n_Lucene50_0.doc, _15n_Lucene50_0.tim, _15n_Lucene50_0.tip, _15o.fdt, _15o.fdx, _15o.fnm, _15o.nvd, _15o.nvm, _15o.si, _15o_Lucene50_0.doc, _15o_Lucene50_0.tim, _15o_Lucene50_0.tip, _15p.fdt, _15p.fdx, _15p.fnm, _15p.nvd, _15p.nvm, _15p.si, _15p_Lucene50_0.doc, _15p_Lucene50_0.tim, _15p_Lucene50_0.tip, _15q.fdt, _15q.fdx, _15q.fnm, _15q.nvd, _15q.nvm, _15q.si, _15q_Lucene50_0.doc, _15q_Lucene50_0.tim, _15q_Lucene50_0.tip, _15r.fdt, _15r.fdx, _15r.fnm, _15r.nvd, _15r.nvm, _15r.si, _15r_Lucene50_0.doc, _15r_Lucene50_0.tim, _15r_Lucene50_0.tip, _15s.fdt, _15s.fdx, _15s.fnm, _15s.nvd, _15s.nvm, _15s.si, _15s_Lucene50_0.doc, _15s_Lucene50_0.tim, _15s_Lucene50_0.tip, _15t.fdt, _15t.fdx, _15t.fnm, _15t.nvd, _15t.nvm, _15t.si, _15t_Lucene50_0.doc, _15t_Lucene50_0.tim, _15t_Lucene50_0.tip, _15u.fdt, _15u.fdx, _15u.fnm, _15u.nvd, _15u.nvm, _15u.si, _15u_Lucene50_0.doc, _15u_Lucene50_0.tim, _15u_Lucene50_0.tip, _15v.fdt, _15v.fdx, _15v.fnm, _15v.nvd, _15v.nvm, _15v.si, _15v_Lucene50_0.doc, _15v_Lucene50_0.tim, _15v_Lucene50_0.tip, _15w.fdt, _15w.fdx, _15w.fnm, _15w.nvd, _15w.nvm, _15w.si, _15w_Lucene50_0.doc, _15w_Lucene50_0.tim, _15w_Lucene50_0.tip, _15x.fdt, _15x.fdx, _15x.fnm, _15x.nvd, _15x.nvm, _15x.si, _15x_Lucene50_0.doc, _15x_Lucene50_0.tim, _15x_Lucene50_0.tip, _15y.fdt, _15y.fdx, _15y.fnm, _15y.nvd, _15y.nvm, _15y.si, _15y_Lucene50_0.doc, _15y_Lucene50_0.tim, _15y_Lucene50_0.tip, _15z.fdt, _15z.fdx, _15z.fnm, _15z.nvd, _15z.nvm, _15z.si, _15z_Lucene50_0.doc, _15z_Lucene50_0.tim, _15z_Lucene50_0.tip, _161.fdt, _161.fdx, _161.fnm, _161.nvd, _161.nvm, _161.si, _161_Lucene50_0.doc, _161_Lucene50_0.tim, _161_Lucene50_0.tip, _162.fdt, _162.fdx, _162.fnm, _162.nvd, _162.nvm, _162.si, _162_Lucene50_0.doc, _162_Lucene50_0.tim, _162_Lucene50_0.tip, _163.fdt, _163.fdx, _163.fnm, _163.nvd, _163.nvm, _163.si, _163_Lucene50_0.doc, _163_Lucene50_0.tim, _163_Lucene50_0.tip, _164.fdt, _164.fdx, _164.fnm, _164.nvd, _164.nvm, _164.si, _164_Lucene50_0.doc, _164_Lucene50_0.tim, _164_Lucene50_0.tip, _165.fdt, _165.fdx, _165.fnm, _165.nvd, _165.nvm, _165.si, _165_Lucene50_0.doc, _165_Lucene50_0.tim, _165_Lucene50_0.tip, _166.fdt, _166.fdx, _166.fnm, _166.nvd, _166.nvm, _166.si, _166_Lucene50_0.doc, _166_Lucene50_0.tim, _166_Lucene50_0.tip, _167.fdt, _167.fdx, _167.fnm, _167.nvd, _167.nvm, _167.si, _167_Lucene50_0.doc, _167_Lucene50_0.tim, _167_Lucene50_0.tip, _168.fdt, _168.fdx, _168.fnm, _168.nvd, _168.nvm, _168.si, _168_Lucene50_0.doc, _168_Lucene50_0.tim, _168_Lucene50_0.tip, _169.fdt, _169.fdx, _169.fnm, _169.nvd, _169.nvm, _169.si, _169_Lucene50_0.doc, _169_Lucene50_0.tim, _169_Lucene50_0.tip, _16a.fdt, _16a.fdx, _16a.fnm, _16a.nvd, _16a.nvm, _16a.si, _16a_Lucene50_0.doc, _16a_Lucene50_0.tim, _16a_Lucene50_0.tip, _16b.fdt, _16b.fdx, _16b.fnm, _16b.nvd, _16b.nvm, _16b.si, _16b_Lucene50_0.doc, _16b_Lucene50_0.tim, _16b_Lucene50_0.tip, _16c.fdt, _16c.fdx, _16c.fnm, _16c.nvd, _16c.nvm, _16c.si, _16c_Lucene50_0.doc, _16c_Lucene50_0.tim, _16c_Lucene50_0.tip, _16d.fdt, _16d.fdx, _16d.fnm, _16d.nvd, _16d.nvm, _16d.si, _16d_Lucene50_0.doc, _16d_Lucene50_0.tim, _16d_Lucene50_0.tip, _16e.fdt, _16e.fdx, _16e.fnm, _16e.nvd, _16e.nvm, _16e.si, _16e_Lucene50_0.doc, _16e_Lucene50_0.tim, _16e_Lucene50_0.tip, _16f.fdt, _16f.fdx, _16f.fnm, _16f.nvd, _16f.nvm, _16f.si, _16f_Lucene50_0.doc, _16f_Lucene50_0.tim, _16f_Lucene50_0.tip, _16g.fdt, _16g.fdx, _16g.fnm, _16g.nvd, _16g.nvm, _16g.si, _16g_Lucene50_0.doc, _16g_Lucene50_0.tim, _16g_Lucene50_0.tip, _16h.fdt, _16h.fdx, _16h.fnm, _16h.nvd, _16h.nvm, _16h.si, _16h_Lucene50_0.doc, _16h_Lucene50_0.tim, _16h_Lucene50_0.tip, _16i.fdt, _16i.fdx, _16i.fnm, _16i.nvd, _16i.nvm, _16i.si, _16i_Lucene50_0.doc, _16i_Lucene50_0.tim, _16i_Lucene50_0.tip, _16j.fdt, _16j.fdx, _16j.fnm, _16j.nvd, _16j.nvm, _16j.si, _16j_Lucene50_0.doc, _16j_Lucene50_0.tim, _16j_Lucene50_0.tip, segments_2]}, {indexVersion=1521590816599,generation=3,filelist=[_160.fdt, _160.fdx, _160.fnm, _160.nvd, _160.nvm, _160.si, _160_Lucene50_0.doc, _160_Lucene50_0.tim, _160_Lucene50_0.tip, _161.fdt, _161.fdx, _161.fnm, _161.nvd, _161.nvm, _161.si, _161_Lucene50_0.doc, _161_Lucene50_0.tim, _161_Lucene50_0.tip, _162.fdt, _162.fdx, _162.fnm, _162.nvd, _162.nvm, _162.si, _162_Lucene50_0.doc, _162_Lucene50_0.tim, _162_Lucene50_0.tip, _163.fdt, _163.fdx, _163.fnm, _163.nvd, _163.nvm, _163.si, _163_Lucene50_0.doc, _163_Lucene50_0.tim, _163_Lucene50_0.tip, _164.fdt, _164.fdx, _164.fnm, _164.nvd, _164.nvm, _164.si, _164_Lucene50_0.doc, _164_Lucene50_0.tim, _164_Lucene50_0.tip, _165.fdt, _165.fdx, _165.fnm, _165.nvd, _165.nvm, _165.si, _165_Lucene50_0.doc, _165_Lucene50_0.tim, _165_Lucene50_0.tip, _166.fdt, _166.fdx, _166.fnm, _166.nvd, _166.nvm, _166.si, _166_Lucene50_0.doc, _166_Lucene50_0.tim, _166_Lucene50_0.tip, _167.fdt, _167.fdx, _167.fnm, _167.nvd, _167.nvm, _167.si, _167_Lucene50_0.doc, _167_Lucene50_0.tim, _167_Lucene50_0.tip, _168.fdt, _168.fdx, _168.fnm, _168.nvd, _168.nvm, _168.si, _168_Lucene50_0.doc, _168_Lucene50_0.tim, _168_Lucene50_0.tip, _169.fdt, _169.fdx, _169.fnm, _169.nvd, _169.nvm, _169.si, _169_Lucene50_0.doc, _169_Lucene50_0.tim, _169_Lucene50_0.tip, _16a.fdt, _16a.fdx, _16a.fnm, _16a.nvd, _16a.nvm, _16a.si, _16a_Lucene50_0.doc, _16a_Lucene50_0.tim, _16a_Lucene50_0.tip, _16b.fdt, _16b.fdx, _16b.fnm, _16b.nvd, _16b.nvm, _16b.si, _16b_Lucene50_0.doc, _16b_Lucene50_0.tim, _16b_Lucene50_0.tip, _16c.fdt, _16c.fdx, _16c.fnm, _16c.nvd, _16c.nvm, _16c.si, _16c_Lucene50_0.doc, _16c_Lucene50_0.tim, _16c_Lucene50_0.tip, _16d.fdt, _16d.fdx, _16d.fnm, _16d.nvd, _16d.nvm, _16d.si, _16d_Lucene50_0.doc, _16d_Lucene50_0.tim, _16d_Lucene50_0.tip, _16e.fdt, _16e.fdx, _16e.fnm, _16e.nvd, _16e.nvm, _16e.si, _16e_Lucene50_0.doc, _16e_Lucene50_0.tim, _16e_Lucene50_0.tip, _16f.fdt, _16f.fdx, _16f.fnm, _16f.nvd, _16f.nvm, _16f.si, _16f_Lucene50_0.doc, _16f_Lucene50_0.tim, _16f_Lucene50_0.tip, _16g.fdt, _16g.fdx, _16g.fnm, _16g.nvd, _16g.nvm, _16g.si, _16g_Lucene50_0.doc, _16g_Lucene50_0.tim, _16g_Lucene50_0.tip, _16h.fdt, _16h.fdx, _16h.fnm, _16h.nvd, _16h.nvm, _16h.si, _16h_Lucene50_0.doc, _16h_Lucene50_0.tim, _16h_Lucene50_0.tip, _16i.fdt, _16i.fdx, _16i.fnm, _16i.nvd, _16i.nvm, _16i.si, _16i_Lucene50_0.doc, _16i_Lucene50_0.tim, _16i_Lucene50_0.tip, _16j.fdt, _16j.fdx, _16j.fnm, _16j.nvd, _16j.nvm, _16j.si, _16j_Lucene50_0.doc, _16j_Lucene50_0.tim, _16j_Lucene50_0.tip, segments_3]}]>
>         at __randomizedtesting.SeedInfo.seed([8C1E58940135899E:A9C943A4717D879D]:0)
>         at org.junit.Assert.fail(Assert.java:93)
>         at org.junit.Assert.failNotEquals(Assert.java:647)
>         at org.junit.Assert.assertEquals(Assert.java:128)
>         at org.junit.Assert.assertEquals(Assert.java:147)
>         at org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload(TestReplicationHandler.java:1284)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:498)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
>         at java.lang.Thread.run(Thread.java:748)
>
>
> FAILED:  org.apache.solr.handler.admin.AutoscalingHistoryHandlerTest.testHistory
>
> Error Message:
> expected:<8> but was:<10>
>
> Stack Trace:
> java.lang.AssertionError: expected:<8> but was:<10>
>         at __randomizedtesting.SeedInfo.seed([8C1E58940135899E:E1E2FC69BB7D7699]:0)
>         at org.junit.Assert.fail(Assert.java:93)
>         at org.junit.Assert.failNotEquals(Assert.java:647)
>         at org.junit.Assert.assertEquals(Assert.java:128)
>         at org.junit.Assert.assertEquals(Assert.java:472)
>         at org.junit.Assert.assertEquals(Assert.java:456)
>         at org.apache.solr.handler.admin.AutoscalingHistoryHandlerTest.testHistory(AutoscalingHistoryHandlerTest.java:354)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:498)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
>         at java.lang.Thread.run(Thread.java:748)
>
>
> FAILED:  org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates.test_dv
>
> Error Message:
> java.lang.RuntimeException: Could not load collection from ZK: test_col
>
> Stack Trace:
> java.util.concurrent.ExecutionException: java.lang.RuntimeException: Could not load collection from ZK: test_col
>         at __randomizedtesting.SeedInfo.seed([8C1E58940135899E:BA0A3AD28B68B38F]:0)
>         at java.util.concurrent.FutureTask.report(FutureTask.java:122)
>         at java.util.concurrent.FutureTask.get(FutureTask.java:192)
>         at org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates.checkField(TestStressCloudBlindAtomicUpdates.java:288)
>         at org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates.test_dv(TestStressCloudBlindAtomicUpdates.java:198)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:498)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
>         at java.lang.Thread.run(Thread.java:748)
> Caused by: java.lang.RuntimeException: Could not load collection from ZK: test_col
>         at org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates$Worker.run(TestStressCloudBlindAtomicUpdates.java:416)
>         at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
>         at java.util.concurrent.FutureTask.run(FutureTask.java:266)
>         at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
>         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
>         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
>         ... 1 more
> Caused by: org.apache.solr.common.SolrException: Could not load collection from ZK: test_col
>         at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1236)
>         at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:674)
>         at org.apache.solr.common.cloud.ClusterState$CollectionRef.get(ClusterState.java:386)
>         at org.apache.solr.client.solrj.impl.CloudSolrClient.getDocCollection(CloudSolrClient.java:1207)
>         at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:850)
>         at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:818)
>         at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
>         at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
>         at org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates$Worker.doRandomAtomicUpdate(TestStressCloudBlindAtomicUpdates.java:375)
>         at org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates$Worker.run(TestStressCloudBlindAtomicUpdates.java:404)
>         ... 6 more
> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/test_col/state.json
>         at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
>         at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
>         at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
>         at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
>         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
>         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
>         at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1248)
>         at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1234)
>         ... 15 more
>
>
> FAILED:  org.apache.solr.cloud.hdfs.HdfsUnloadDistributedZkTest.test
>
> Error Message:
> Captured an uncaught exception in thread: Thread[id=2892, name=testExecutor-1036-thread-18, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest]
>
> Stack Trace:
> com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=2892, name=testExecutor-1036-thread-18, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest]
> Caused by: java.lang.RuntimeException: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:36627/p_cib/er
>         at __randomizedtesting.SeedInfo.seed([8C1E58940135899E]:0)
>         at org.apache.solr.cloud.BasicDistributedZkTest.lambda$createCollectionInOneInstance$0(BasicDistributedZkTest.java:596)
>         at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
>         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
>         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
>         at java.lang.Thread.run(Thread.java:748)
> Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:36627/p_cib/er
>         at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:654)
>         at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
>         at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
>         at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
>         at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
>         at org.apache.solr.cloud.BasicDistributedZkTest.lambda$createCollectionInOneInstance$0(BasicDistributedZkTest.java:594)
>         ... 4 more
> Caused by: java.net.SocketTimeoutException: Read timed out
>         at java.net.SocketInputStream.socketRead0(Native Method)
>         at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
>         at java.net.SocketInputStream.read(SocketInputStream.java:171)
>         at java.net.SocketInputStream.read(SocketInputStream.java:141)
>         at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
>         at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
>         at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
>         at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
>         at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
>         at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
>         at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
>         at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
>         at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
>         at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
>         at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
>         at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
>         at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
>         at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
>         at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
>         at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
>         at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
>         at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:542)
>         ... 9 more
>
>
>
>
> Build Log:
> [...truncated 2029 lines...]
>    [junit4] Suite: org.apache.lucene.index.TestIndexWriterOnVMError
>    [junit4]   1> Unexpected exception: dumping fake-exception-log:...
>    [junit4]   1>
>    [junit4]   1> TEST: got expected fake exc:Fake UnknownError
>    [junit4]   1> java.lang.UnknownError: Fake UnknownError
>    [junit4]   1>        at org.apache.lucene.index.TestIndexWriterOnVMError$3.eval(TestIndexWriterOnVMError.java:270)
>    [junit4]   1>        at org.apache.lucene.store.MockDirectoryWrapper.maybeThrowDeterministicException(MockDirectoryWrapper.java:1022)
>    [junit4]   1>        at org.apache.lucene.store.MockIndexOutputWrapper.writeBytes(MockIndexOutputWrapper.java:144)
>    [junit4]   1>        at org.apache.lucene.store.MockIndexOutputWrapper.writeByte(MockIndexOutputWrapper.java:126)
>    [junit4]   1>        at org.apache.lucene.store.DataOutput.writeInt(DataOutput.java:73)
>    [junit4]   1>        at org.apache.lucene.store.DataOutput.writeLong(DataOutput.java:213)
>    [junit4]   1>        at org.apache.lucene.codecs.CodecUtil.writeCRC(CodecUtil.java:552)
>    [junit4]   1>        at org.apache.lucene.codecs.CodecUtil.writeFooter(CodecUtil.java:393)
>    [junit4]   1>        at org.apache.lucene.codecs.blockterms.BlockTermsWriter.close(BlockTermsWriter.java:183)
>    [junit4]   1>        at org.apache.lucene.util.IOUtils.close(IOUtils.java:89)
>    [junit4]   1>        at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.close(PerFieldPostingsFormat.java:241)
>    [junit4]   1>        at org.apache.lucene.util.IOUtils.close(IOUtils.java:89)
>    [junit4]   1>        at org.apache.lucene.util.IOUtils.close(IOUtils.java:76)
>    [junit4]   1>        at org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:112)
>    [junit4]   1>        at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:163)
>    [junit4]   1>        at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:463)
>    [junit4]   1>        at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:556)
>    [junit4]   1>        at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:672)
>    [junit4]   1>        at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:454)
>    [junit4]   1>        at org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:103)
>    [junit4]   1>        at org.apache.lucene.index.TestIndexWriterOnVMError.doTest(TestIndexWriterOnVMError.java:175)
>    [junit4]   1>        at org.apache.lucene.index.TestIndexWriterOnVMError.testUnknownError(TestIndexWriterOnVMError.java:258)
>    [junit4]   1>        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>    [junit4]   1>        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
>    [junit4]   1>        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>    [junit4]   1>        at java.lang.reflect.Method.invoke(Method.java:498)
>    [junit4]   1>        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
>    [junit4]   1>        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
>    [junit4]   1>        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
>    [junit4]   1>        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
>    [junit4]   1>        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
>    [junit4]   1>        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
>    [junit4]   1>        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
>    [junit4]   1>        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
>    [junit4]   1>        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
>    [junit4]   1>        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>    [junit4]   1>        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
>    [junit4]   1>        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
>    [junit4]   1>        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
>    [junit4]   1>        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
>    [junit4]   1>        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
>    [junit4]   1>        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
>    [junit4]   1>        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
>    [junit4]   1>        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
>    [junit4]   1>        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>    [junit4]   1>        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
>    [junit4]   1>        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
>    [junit4]   1>        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
>    [junit4]   1>        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>    [junit4]   1>        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>    [junit4]   1>        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>    [junit4]   1>        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
>    [junit4]   1>        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
>    [junit4]   1>        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
>    [junit4]   1>        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
>    [junit4]   1>        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>    [junit4]   1>        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
>    [junit4]   1>        at java.lang.Thread.run(Thread.java:748)
>    [junit4]   1>
>    [junit4]   1> TEST: got expected fake exc:Fake UnknownError
>    [junit4]   1> java.lang.UnknownError: Fake UnknownError
>    [junit4]   1>        at org.apache.lucene.index.TestIndexWriterOnVMError$3.eval(TestIndexWriterOnVMError.java:270)
>    [junit4]   1>        at org.apache.lucene.store.MockDirectoryWrapper.maybeThrowDeterministicException(MockDirectoryWrapper.java:1022)
>    [junit4]   1>        at org.apache.lucene.store.MockIndexOutputWrapper.writeBytes(MockIndexOutputWrapper.java:144)
>    [junit4]   1>        at org.apache.lucene.store.MockIndexOutputWrapper.writeByte(MockIndexOutputWrapper.java:126)
>    [junit4]   1>        at org.apache.lucene.store.DataOutput.writeInt(DataOutput.java:70)
>    [junit4]   1>        at org.apache.lucene.store.DataOutput.writeLong(DataOutput.java:213)
>    [junit4]   1>        at org.apache.lucene.codecs.lucene70.Lucene70DocValuesConsumer.addTermsDict(Lucene70DocValuesConsumer.java:480)
>    [junit4]   1>        at org.apache.lucene.codecs.lucene70.Lucene70DocValuesConsumer.doAddSortedField(Lucene70DocValuesConsumer.java:437)
>    [junit4]   1>        at org.apache.lucene.codecs.lucene70.Lucene70DocValuesConsumer.addSortedField(Lucene70DocValuesConsumer.java:394)
>    [junit4]   1>        at org.apache.lucene.codecs.asserting.AssertingDocValuesFormat$AssertingDocValuesConsumer.addSortedField(AssertingDocValuesFormat.java:141)
>    [junit4]   1>        at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addSortedField(PerFieldDocValuesFormat.java:119)
>    [junit4]   1>        at org.apache.lucene.index.SortedDocValuesWriter.flush(SortedDocValuesWriter.java:163)
>    [junit4]   1>        at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:248)
>    [junit4]   1>        at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:132)
>    [junit4]   1>        at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:463)
>    [junit4]   1>        at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:556)
>    [junit4]   1>        at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:672)
>    [junit4]   1>        at org.apache.lucene.index.IndexWriter.prepareCommitInternal(IndexWriter.java:3337)
>    [junit4]   1>        at org.apache.lucene.index.IndexWriter.commitInternal(IndexWriter.java:3549)
>    [junit4]   1>        at org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3514)
>    [junit4]   1>        at org.apache.lucene.index.TestIndexWriterOnVMError.doTest(TestIndexWriterOnVMError.java:181)
>    [junit4]   1>        at org.apache.lucene.index.TestIndexWriterOnVMError.testUnknownError(TestIndexWriterOnVMError.java:258)
>    [junit4]   1>        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>    [junit4]   1>        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
>    [junit4]   1>        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>    [junit4]   1>        at java.lang.reflect.Method.invoke(Method.java:498)
>    [junit4]   1>        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
>    [junit4]   1>        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
>    [junit4]   1>        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
>    [junit4]   1>        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
>    [junit4]   1>        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
>    [junit4]   1>        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
>    [junit4]   1>        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
>    [junit4]   1>        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
>    [junit4]   1>        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
>    [junit4]   1>        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>    [junit4]   1>        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
>    [junit4]   1>        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
>    [junit4]   1>        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
>    [junit4]   1>        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
>    [junit4]   1>        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
>    [junit4]   1>        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
>    [junit4]   1>        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
>    [junit4]   1>        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
>    [junit4]   1>        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>    [junit4]   1>        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
>    [junit4]   1>        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
>    [junit4]   1>        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
>    [junit4]   1>        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>    [junit4]   1>        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>    [junit4]   1>        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>    [junit4]   1>        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
>    [junit4]   1>        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
>    [junit4]   1>        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
>    [junit4]   1>        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
>    [junit4]   1>        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>    [junit4]   1>        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
>    [junit4]   1>        at java.lang.Thread.run(Thread.java:748)
>    [junit4]   1>
>    [junit4]   1> TEST: got expected fake exc:Fake UnknownError
>    [junit4]   1> java.lang.UnknownError: Fake UnknownError
>    [junit4]   1>        at org.apache.lucene.index.TestIndexWriterOnVMError$3.eval(TestIndexWriterOnVMError.java:270)
>    [junit4]   1>        at org.apa
>
> [...truncated too long message...]
>
> EST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[8C1E58940135899E]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
>    [junit4]   2> 9158573 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[8C1E58940135899E]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@3284d0f1: rootName = null, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@1d5100
>    [junit4]   2> 9158576 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[8C1E58940135899E]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
>    [junit4]   2> 9158577 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[8C1E58940135899E]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@4672aa43: rootName = null, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@5cc3c634
>    [junit4]   2> 9158577 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[8C1E58940135899E]) [    ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:39036_
>    [junit4]   2> 9158578 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[8C1E58940135899E]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
>    [junit4]   2> 9158579 INFO  (zkCallback-10620-thread-1-processing-n:127.0.0.1:39036_) [n:127.0.0.1:39036_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
>    [junit4]   2> 9158579 INFO  (zkCallback-10625-thread-1-processing-n:127.0.0.1:39036_) [n:127.0.0.1:39036_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
>    [junit4]   2> 9158580 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[8C1E58940135899E]) [    ] o.a.s.c.Overseer Overseer (id=72315250603458599-127.0.0.1:39036_-n_0000000007) closing
>    [junit4]   2> 9158580 INFO  (OverseerStateUpdate-72315250603458599-127.0.0.1:39036_-n_0000000007) [n:127.0.0.1:39036_    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:39036_
>    [junit4]   2> 9158580 WARN  (OverseerAutoScalingTriggerThread-72315250603458599-127.0.0.1:39036_-n_0000000007) [n:127.0.0.1:39036_    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
>    [junit4]   2> 9158583 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[8C1E58940135899E]) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@306c6875{SSL,[ssl, http/1.1]}{127.0.0.1:0}
>    [junit4]   2> 9158584 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[8C1E58940135899E]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@52092334{/,null,UNAVAILABLE}
>    [junit4]   2> 9158584 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[8C1E58940135899E]) [    ] o.e.j.s.session Stopped scavenging
>    [junit4]   2> 9158584 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[8C1E58940135899E]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testBatchAddsWithDelete
>    [junit4]   2> 9158584 ERROR (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[8C1E58940135899E]) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
>    [junit4]   2> 9158585 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[8C1E58940135899E]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:37459 37459
>    [junit4]   2> 9158654 INFO  (Thread-64967) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:37459 37459
>    [junit4]   2> 9158655 WARN  (Thread-64967) [    ] o.a.s.c.ZkTestServer Watch limit violations:
>    [junit4]   2> Maximum concurrent create/delete watches above limit:
>    [junit4]   2>
>    [junit4]   2>        53      /solr/collections/tmp_collection/terms/shard2
>    [junit4]   2>        52      /solr/collections/tmp_collection/terms/shard1
>    [junit4]   2>        38      /solr/aliases.json
>    [junit4]   2>        14      /solr/configs/conf1
>    [junit4]   2>        8       /solr/security.json
>    [junit4]   2>        8       /solr/collections/target_collection/terms/shard1
>    [junit4]   2>        7       /solr/collections/source_collection/terms/shard1
>    [junit4]   2>        7       /solr/collections/source_collection/terms/shard2
>    [junit4]   2>        7       /solr/collections/target_collection/terms/shard2
>    [junit4]   2>        4       /solr/collections/tmp_collection/leaders/shard2/leader
>    [junit4]   2>        4       /solr/collections/tmp_collection/leaders/shard1/leader
>    [junit4]   2>        2       /solr/collections/source_collection/leaders/shard2/leader
>    [junit4]   2>        2       /solr/collections/source_collection/leaders/shard1/leader
>    [junit4]   2>        2       /solr/collections/target_collection/leaders/shard2/leader
>    [junit4]   2>        2       /solr/collections/target_collection/leaders/shard1/leader
>    [junit4]   2>
>    [junit4]   2> Maximum concurrent data watches above limit:
>    [junit4]   2>
>    [junit4]   2>        38      /solr/clusterprops.json
>    [junit4]   2>        38      /solr/clusterstate.json
>    [junit4]   2>        12      /solr/collections/tmp_collection/cdcr/state/process
>    [junit4]   2>        12      /solr/collections/tmp_collection/state.json
>    [junit4]   2>        12      /solr/collections/tmp_collection/cdcr/state/buffer
>    [junit4]   2>        7       /solr/autoscaling.json
>    [junit4]   2>        5       /solr/collections/source_collection/cdcr/state/process
>    [junit4]   2>        4       /solr/collections/target_collection/cdcr/state/buffer
>    [junit4]   2>        4       /solr/collections/source_collection/state.json
>    [junit4]   2>        4       /solr/collections/source_collection/cdcr/state/buffer
>    [junit4]   2>        4       /solr/collections/target_collection/cdcr/state/process
>    [junit4]   2>        4       /solr/collections/target_collection/state.json
>    [junit4]   2>        3       /solr/collections/tmp_collection/leader_elect/shard1/election/72315250603458565-core_node42-n_0000000000
>    [junit4]   2>        2       /solr/collections/tmp_collection/leader_elect/shard2/election/72315250603458569-core_node44-n_0000000000
>    [junit4]   2>
>    [junit4]   2> Maximum concurrent children watches above limit:
>    [junit4]   2>
>    [junit4]   2>        38      /solr/live_nodes
>    [junit4]   2>        37      /solr/collections
>    [junit4]   2>        7       /solr/overseer/queue
>    [junit4]   2>        7       /solr/autoscaling/events/.scheduled_maintenance
>    [junit4]   2>        7       /solr/overseer/collection-queue-work
>    [junit4]   2>        7       /solr/overseer/queue-work
>    [junit4]   2>        6       /solr/autoscaling/events/.auto_add_replicas
>    [junit4]   2>
>    [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrReplicationDistributedZkTest_8C1E58940135899E-001
>    [junit4]   2> Mar 21, 2018 1:03:15 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
>    [junit4]   2> WARNING: Will linger awaiting termination of 5 leaked thread(s).
>    [junit4]   2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarity(queryNorm=true): {}, locale=lt, timezone=Pacific/Pago_Pago
>    [junit4]   2> NOTE: Linux 4.4.0-112-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=1,free=66199328,total=521666560
>    [junit4]   2> NOTE: All tests run in this JVM: [TestCloudPivotFacet, MultiTermTest, MinimalSchemaTest, TestBadConfig, HdfsCollectionsAPIDistributedZkTest, DeleteShardTest, TestRetrieveFieldsOptimizer, UniqFieldsUpdateProcessorFactoryTest, URLClassifyProcessorTest, DirectSolrSpellCheckerTest, TestLegacyFieldReuse, TestHalfAndHalfDocValues, ZkFailoverTest, TestIntervalFaceting, DeleteLastCustomShardedReplicaTest, TestAddFieldRealTimeGet, HdfsBasicDistributedZkTest, TimeZoneUtilsTest, TestSolrCloudWithKerberosAlt, ConvertedLegacyTest, DirectoryFactoryTest, ComputePlanActionTest, PingRequestHandlerTest, TestDownShardTolerantSearch, TestDistribIDF, HdfsTlogReplayBufferedWhileIndexingTest, TestManagedResource, SolrTestCaseJ4Test, TestDFRSimilarityFactory, MetricsConfigTest, TestUtils, DistributedFacetPivotLongTailTest, SolrMetricReporterTest, TestSolrConfigHandlerCloud, HdfsRestartWhileUpdatingTest, TestRealTimeGet, TestMultiValuedNumericRangeQuery, DistributedSuggestComponentTest, TestDistribStateManager, RulesTest, TestMiniSolrCloudClusterSSL, TestApiFramework, TestAnalyzedSuggestions, TestQueryWrapperFilter, NotRequiredUniqueKeyTest, BooleanFieldTest, TestJettySolrRunner, PKIAuthenticationIntegrationTest, TestReRankQParserPlugin, DocValuesMultiTest, UUIDUpdateProcessorFallbackTest, SharedFSAutoReplicaFailoverTest, DistanceFunctionTest, StatelessScriptUpdateProcessorFactoryTest, V2ApiIntegrationTest, UpdateLogTest, RegexBytesRefFilterTest, TestRandomRequestDistribution, TestCoreAdminApis, SuggestComponentContextFilterQueryTest, TestEmbeddedSolrServerAdminHandler, TestSolrDeletionPolicy2, TestLFUCache, TestPullReplica, CdcrVersionReplicationTest, TestClassicSimilarityFactory, CircularListTest, OverseerStatusTest, TestNamedUpdateProcessors, PreAnalyzedUpdateProcessorTest, TestMacros, CopyFieldTest, SearchRateTriggerTest, UnloadDistributedZkTest, CdcrUpdateLogTest, BlockJoinFacetSimpleTest, TestAtomicUpdateErrorCases, RollingRestartTest, ClusterStateTest, TestSuggestSpellingConverter, TestLeaderElectionZkExpiry, TestStressInPlaceUpdates, AutoAddReplicasIntegrationTest, HdfsNNFailoverTest, CacheHeaderTest, TestUninvertingReader, TestRestoreCore, TestSolrXml, TestFieldTypeCollectionResource, TestCoreDiscovery, TestSSLRandomization, SuggesterTSTTest, TestSerializedLuceneMatchVersion, ReplaceNodeTest, SolrSlf4jReporterTest, TestCloudDeleteByQuery, HdfsDirectoryTest, DisMaxRequestHandlerTest, HttpSolrCallGetCoreTest, CdcrReplicationDistributedZkTest]
>    [junit4] Completed [613/784 (8!)] on J2 in 1836.16s, 11 tests, 1 failure <<< FAILURES!
>
> [...truncated 50059 lines...]
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org

---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org


[JENKINS] Lucene-Solr-BadApples-NightlyTests-7.x - Build # 3 - Still Unstable

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-BadApples-NightlyTests-7.x/3/

10 tests failed.
FAILED:  org.apache.lucene.index.TestIndexWriterOnVMError.testUnknownError

Error Message:
MockDirectoryWrapper: cannot close: there are still 1 open files: {_0_Asserting_0.dvm=1}

Stack Trace:
java.lang.RuntimeException: MockDirectoryWrapper: cannot close: there are still 1 open files: {_0_Asserting_0.dvm=1}
	at __randomizedtesting.SeedInfo.seed([C1B6EA2337948F2A:2AE035A33513191A]:0)
	at org.apache.lucene.store.MockDirectoryWrapper.close(MockDirectoryWrapper.java:841)
	at org.apache.lucene.index.TestIndexWriterOnVMError.doTest(TestIndexWriterOnVMError.java:89)
	at org.apache.lucene.index.TestIndexWriterOnVMError.testUnknownError(TestIndexWriterOnVMError.java:258)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.RuntimeException: unclosed IndexOutput: _0_Asserting_0.dvm
	at org.apache.lucene.store.MockDirectoryWrapper.addFileHandle(MockDirectoryWrapper.java:732)
	at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:667)
	at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44)
	at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
	at org.apache.lucene.codecs.lucene70.Lucene70DocValuesConsumer.<init>(Lucene70DocValuesConsumer.java:72)
	at org.apache.lucene.codecs.lucene70.Lucene70DocValuesFormat.fieldsConsumer(Lucene70DocValuesFormat.java:127)
	at org.apache.lucene.codecs.asserting.AssertingDocValuesFormat.fieldsConsumer(AssertingDocValuesFormat.java:55)
	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:211)
	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addSortedField(PerFieldDocValuesFormat.java:119)
	at org.apache.lucene.index.SortedDocValuesWriter.flush(SortedDocValuesWriter.java:163)
	at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:248)
	at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:132)
	at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:463)
	at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:556)
	at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:672)
	at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:454)
	at org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:103)
	at org.apache.lucene.index.TestIndexWriterOnVMError.doTest(TestIndexWriterOnVMError.java:175)
	... 37 more


FAILED:  org.apache.solr.cloud.LIROnShardRestartTest.testAllReplicasInLIR

Error Message:
No live SolrServers available to handle this request

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request
	at __randomizedtesting.SeedInfo.seed([8C1E58940135899E:D68662527FB5EE79]:0)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:460)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1105)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:885)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:818)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
	at org.apache.solr.cloud.LIROnShardRestartTest.testAllReplicasInLIR(LIROnShardRestartTest.java:160)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.LIROnShardRestartTest.testSeveralReplicasInLIR

Error Message:
No live SolrServers available to handle this request

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request
	at __randomizedtesting.SeedInfo.seed([8C1E58940135899E:A44CFA0216FC4C7]:0)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:460)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1105)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:885)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:818)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
	at org.apache.solr.cloud.LIROnShardRestartTest.testSeveralReplicasInLIR(LIROnShardRestartTest.java:175)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test

Error Message:
Expected numSlices=5 numReplicas=1 but found DocCollection(solrj_collection4//collections/solrj_collection4/state.json/33)={   "pullReplicas":"0",   "replicationFactor":"1",   "shards":{     "shard1":{       "range":"80000000-b332ffff",       "state":"active",       "replicas":{"core_node3":{           "dataDir":"hdfs://lucene2-us-west.apache.org:39207/solr_hdfs_home/solrj_collection4/core_node3/data/",           "base_url":"http://127.0.0.1:40786/ml_/hn",           "node_name":"127.0.0.1:40786_ml_%2Fhn",           "type":"NRT",           "ulogDir":"hdfs://lucene2-us-west.apache.org:39207/solr_hdfs_home/solrj_collection4/core_node3/data/tlog",           "core":"solrj_collection4_shard1_replica_n1",           "shared_storage":"true",           "state":"active",           "leader":"true"}}},     "shard2":{       "range":"b3330000-e665ffff",       "state":"active",       "replicas":{"core_node5":{           "dataDir":"hdfs://lucene2-us-west.apache.org:39207/solr_hdfs_home/solrj_collection4/core_node5/data/",           "base_url":"http://127.0.0.1:38648/ml_/hn",           "node_name":"127.0.0.1:38648_ml_%2Fhn",           "type":"NRT",           "ulogDir":"hdfs://lucene2-us-west.apache.org:39207/solr_hdfs_home/solrj_collection4/core_node5/data/tlog",           "core":"solrj_collection4_shard2_replica_n2",           "shared_storage":"true",           "state":"active",           "leader":"true"}}},     "shard3":{       "range":"e6660000-1998ffff",       "state":"active",       "replicas":{"core_node7":{           "dataDir":"hdfs://lucene2-us-west.apache.org:39207/solr_hdfs_home/solrj_collection4/core_node7/data/",           "base_url":"http://127.0.0.1:38648/ml_/hn",           "node_name":"127.0.0.1:38648_ml_%2Fhn",           "type":"NRT",           "ulogDir":"hdfs://lucene2-us-west.apache.org:39207/solr_hdfs_home/solrj_collection4/core_node7/data/tlog",           "core":"solrj_collection4_shard3_replica_n4",           "shared_storage":"true",           "state":"active",           "leader":"true"}}},     "shard4":{       "range":"19990000-4ccbffff",       "state":"active",       "replicas":{"core_node9":{           "dataDir":"hdfs://lucene2-us-west.apache.org:39207/solr_hdfs_home/solrj_collection4/core_node9/data/",           "base_url":"http://127.0.0.1:37880/ml_/hn",           "node_name":"127.0.0.1:37880_ml_%2Fhn",           "type":"NRT",           "ulogDir":"hdfs://lucene2-us-west.apache.org:39207/solr_hdfs_home/solrj_collection4/core_node9/data/tlog",           "core":"solrj_collection4_shard4_replica_n6",           "shared_storage":"true",           "state":"active",           "leader":"true"}}},     "shard5":{       "range":"4ccc0000-7fffffff",       "state":"active",       "replicas":{"core_node10":{           "core":"solrj_collection4_shard5_replica_n8",           "base_url":"http://127.0.0.1:37880/ml_/hn",           "node_name":"127.0.0.1:37880_ml_%2Fhn",           "state":"down",           "type":"NRT"}}}},   "router":{     "field":"text",     "name":"compositeId"},   "maxShardsPerNode":"5",   "autoAddReplicas":"true",   "nrtReplicas":"1",   "tlogReplicas":"0"} with /live_nodes: [127.0.0.1:37880_ml_%2Fhn, 127.0.0.1:38648_ml_%2Fhn, 127.0.0.1:38840_ml_%2Fhn, 127.0.0.1:40786_ml_%2Fhn, 127.0.0.1:45388_ml_%2Fhn]

Stack Trace:
java.lang.AssertionError: Expected numSlices=5 numReplicas=1 but found DocCollection(solrj_collection4//collections/solrj_collection4/state.json/33)={
  "pullReplicas":"0",
  "replicationFactor":"1",
  "shards":{
    "shard1":{
      "range":"80000000-b332ffff",
      "state":"active",
      "replicas":{"core_node3":{
          "dataDir":"hdfs://lucene2-us-west.apache.org:39207/solr_hdfs_home/solrj_collection4/core_node3/data/",
          "base_url":"http://127.0.0.1:40786/ml_/hn",
          "node_name":"127.0.0.1:40786_ml_%2Fhn",
          "type":"NRT",
          "ulogDir":"hdfs://lucene2-us-west.apache.org:39207/solr_hdfs_home/solrj_collection4/core_node3/data/tlog",
          "core":"solrj_collection4_shard1_replica_n1",
          "shared_storage":"true",
          "state":"active",
          "leader":"true"}}},
    "shard2":{
      "range":"b3330000-e665ffff",
      "state":"active",
      "replicas":{"core_node5":{
          "dataDir":"hdfs://lucene2-us-west.apache.org:39207/solr_hdfs_home/solrj_collection4/core_node5/data/",
          "base_url":"http://127.0.0.1:38648/ml_/hn",
          "node_name":"127.0.0.1:38648_ml_%2Fhn",
          "type":"NRT",
          "ulogDir":"hdfs://lucene2-us-west.apache.org:39207/solr_hdfs_home/solrj_collection4/core_node5/data/tlog",
          "core":"solrj_collection4_shard2_replica_n2",
          "shared_storage":"true",
          "state":"active",
          "leader":"true"}}},
    "shard3":{
      "range":"e6660000-1998ffff",
      "state":"active",
      "replicas":{"core_node7":{
          "dataDir":"hdfs://lucene2-us-west.apache.org:39207/solr_hdfs_home/solrj_collection4/core_node7/data/",
          "base_url":"http://127.0.0.1:38648/ml_/hn",
          "node_name":"127.0.0.1:38648_ml_%2Fhn",
          "type":"NRT",
          "ulogDir":"hdfs://lucene2-us-west.apache.org:39207/solr_hdfs_home/solrj_collection4/core_node7/data/tlog",
          "core":"solrj_collection4_shard3_replica_n4",
          "shared_storage":"true",
          "state":"active",
          "leader":"true"}}},
    "shard4":{
      "range":"19990000-4ccbffff",
      "state":"active",
      "replicas":{"core_node9":{
          "dataDir":"hdfs://lucene2-us-west.apache.org:39207/solr_hdfs_home/solrj_collection4/core_node9/data/",
          "base_url":"http://127.0.0.1:37880/ml_/hn",
          "node_name":"127.0.0.1:37880_ml_%2Fhn",
          "type":"NRT",
          "ulogDir":"hdfs://lucene2-us-west.apache.org:39207/solr_hdfs_home/solrj_collection4/core_node9/data/tlog",
          "core":"solrj_collection4_shard4_replica_n6",
          "shared_storage":"true",
          "state":"active",
          "leader":"true"}}},
    "shard5":{
      "range":"4ccc0000-7fffffff",
      "state":"active",
      "replicas":{"core_node10":{
          "core":"solrj_collection4_shard5_replica_n8",
          "base_url":"http://127.0.0.1:37880/ml_/hn",
          "node_name":"127.0.0.1:37880_ml_%2Fhn",
          "state":"down",
          "type":"NRT"}}}},
  "router":{
    "field":"text",
    "name":"compositeId"},
  "maxShardsPerNode":"5",
  "autoAddReplicas":"true",
  "nrtReplicas":"1",
  "tlogReplicas":"0"} with /live_nodes: [127.0.0.1:37880_ml_%2Fhn, 127.0.0.1:38648_ml_%2Fhn, 127.0.0.1:38840_ml_%2Fhn, 127.0.0.1:40786_ml_%2Fhn, 127.0.0.1:45388_ml_%2Fhn]
	at __randomizedtesting.SeedInfo.seed([8C1E58940135899E:44A674EAFC9E466]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.assertSliceAndReplicaCount(SharedFSAutoReplicaFailoverTest.java:415)
	at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:274)
	at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:145)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.autoscaling.AutoScalingHandlerTest.testConcurrentUpdates

Error Message:
Captured an uncaught exception in thread: Thread[id=20023, name=Thread-6668, state=RUNNABLE, group=TGRP-AutoScalingHandlerTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=20023, name=Thread-6668, state=RUNNABLE, group=TGRP-AutoScalingHandlerTest]
Caused by: java.lang.AssertionError: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteExecutionException: Error from server at https://127.0.0.1:41129/solr: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
	at __randomizedtesting.SeedInfo.seed([8C1E58940135899E]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.autoscaling.AutoScalingHandlerTest.lambda$testConcurrentUpdates$0(AutoScalingHandlerTest.java:758)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.cdcr.CdcrReplicationDistributedZkTest.testUpdateLogSynchronisation

Error Message:
Timeout while trying to assert update logs @ source_collection

Stack Trace:
java.lang.AssertionError: Timeout while trying to assert update logs @ source_collection
	at __randomizedtesting.SeedInfo.seed([8C1E58940135899E:72710037C315AA8F]:0)
	at org.apache.solr.cloud.cdcr.CdcrReplicationDistributedZkTest.testUpdateLogSynchronisation(CdcrReplicationDistributedZkTest.java:407)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload

Error Message:
expected:<[{indexVersion=1521590816599,generation=2,filelist=[_15f.fdt, _15f.fdx, _15f.fnm, _15f.nvd, _15f.nvm, _15f.si, _15f_Lucene50_0.doc, _15f_Lucene50_0.tim, _15f_Lucene50_0.tip, _15g.fdt, _15g.fdx, _15g.fnm, _15g.nvd, _15g.nvm, _15g.si, _15g_Lucene50_0.doc, _15g_Lucene50_0.tim, _15g_Lucene50_0.tip, _15h.fdt, _15h.fdx, _15h.fnm, _15h.nvd, _15h.nvm, _15h.si, _15h_Lucene50_0.doc, _15h_Lucene50_0.tim, _15h_Lucene50_0.tip, _15i.fdt, _15i.fdx, _15i.fnm, _15i.nvd, _15i.nvm, _15i.si, _15i_Lucene50_0.doc, _15i_Lucene50_0.tim, _15i_Lucene50_0.tip, _15j.fdt, _15j.fdx, _15j.fnm, _15j.nvd, _15j.nvm, _15j.si, _15j_Lucene50_0.doc, _15j_Lucene50_0.tim, _15j_Lucene50_0.tip, _15k.fdt, _15k.fdx, _15k.fnm, _15k.nvd, _15k.nvm, _15k.si, _15k_Lucene50_0.doc, _15k_Lucene50_0.tim, _15k_Lucene50_0.tip, _15l.fdt, _15l.fdx, _15l.fnm, _15l.nvd, _15l.nvm, _15l.si, _15l_Lucene50_0.doc, _15l_Lucene50_0.tim, _15l_Lucene50_0.tip, _15m.fdt, _15m.fdx, _15m.fnm, _15m.nvd, _15m.nvm, _15m.si, _15m_Lucene50_0.doc, _15m_Lucene50_0.tim, _15m_Lucene50_0.tip, _15n.fdt, _15n.fdx, _15n.fnm, _15n.nvd, _15n.nvm, _15n.si, _15n_Lucene50_0.doc, _15n_Lucene50_0.tim, _15n_Lucene50_0.tip, _15o.fdt, _15o.fdx, _15o.fnm, _15o.nvd, _15o.nvm, _15o.si, _15o_Lucene50_0.doc, _15o_Lucene50_0.tim, _15o_Lucene50_0.tip, _15p.fdt, _15p.fdx, _15p.fnm, _15p.nvd, _15p.nvm, _15p.si, _15p_Lucene50_0.doc, _15p_Lucene50_0.tim, _15p_Lucene50_0.tip, _15q.fdt, _15q.fdx, _15q.fnm, _15q.nvd, _15q.nvm, _15q.si, _15q_Lucene50_0.doc, _15q_Lucene50_0.tim, _15q_Lucene50_0.tip, _15r.fdt, _15r.fdx, _15r.fnm, _15r.nvd, _15r.nvm, _15r.si, _15r_Lucene50_0.doc, _15r_Lucene50_0.tim, _15r_Lucene50_0.tip, _15s.fdt, _15s.fdx, _15s.fnm, _15s.nvd, _15s.nvm, _15s.si, _15s_Lucene50_0.doc, _15s_Lucene50_0.tim, _15s_Lucene50_0.tip, _15t.fdt, _15t.fdx, _15t.fnm, _15t.nvd, _15t.nvm, _15t.si, _15t_Lucene50_0.doc, _15t_Lucene50_0.tim, _15t_Lucene50_0.tip, _15u.fdt, _15u.fdx, _15u.fnm, _15u.nvd, _15u.nvm, _15u.si, _15u_Lucene50_0.doc, _15u_Lucene50_0.tim, _15u_Lucene50_0.tip, _15v.fdt, _15v.fdx, _15v.fnm, _15v.nvd, _15v.nvm, _15v.si, _15v_Lucene50_0.doc, _15v_Lucene50_0.tim, _15v_Lucene50_0.tip, _15w.fdt, _15w.fdx, _15w.fnm, _15w.nvd, _15w.nvm, _15w.si, _15w_Lucene50_0.doc, _15w_Lucene50_0.tim, _15w_Lucene50_0.tip, _15x.fdt, _15x.fdx, _15x.fnm, _15x.nvd, _15x.nvm, _15x.si, _15x_Lucene50_0.doc, _15x_Lucene50_0.tim, _15x_Lucene50_0.tip, _15y.fdt, _15y.fdx, _15y.fnm, _15y.nvd, _15y.nvm, _15y.si, _15y_Lucene50_0.doc, _15y_Lucene50_0.tim, _15y_Lucene50_0.tip, _15z.fdt, _15z.fdx, _15z.fnm, _15z.nvd, _15z.nvm, _15z.si, _15z_Lucene50_0.doc, _15z_Lucene50_0.tim, _15z_Lucene50_0.tip, _161.fdt, _161.fdx, _161.fnm, _161.nvd, _161.nvm, _161.si, _161_Lucene50_0.doc, _161_Lucene50_0.tim, _161_Lucene50_0.tip, _162.fdt, _162.fdx, _162.fnm, _162.nvd, _162.nvm, _162.si, _162_Lucene50_0.doc, _162_Lucene50_0.tim, _162_Lucene50_0.tip, _163.fdt, _163.fdx, _163.fnm, _163.nvd, _163.nvm, _163.si, _163_Lucene50_0.doc, _163_Lucene50_0.tim, _163_Lucene50_0.tip, _164.fdt, _164.fdx, _164.fnm, _164.nvd, _164.nvm, _164.si, _164_Lucene50_0.doc, _164_Lucene50_0.tim, _164_Lucene50_0.tip, _165.fdt, _165.fdx, _165.fnm, _165.nvd, _165.nvm, _165.si, _165_Lucene50_0.doc, _165_Lucene50_0.tim, _165_Lucene50_0.tip, _166.fdt, _166.fdx, _166.fnm, _166.nvd, _166.nvm, _166.si, _166_Lucene50_0.doc, _166_Lucene50_0.tim, _166_Lucene50_0.tip, _167.fdt, _167.fdx, _167.fnm, _167.nvd, _167.nvm, _167.si, _167_Lucene50_0.doc, _167_Lucene50_0.tim, _167_Lucene50_0.tip, _168.fdt, _168.fdx, _168.fnm, _168.nvd, _168.nvm, _168.si, _168_Lucene50_0.doc, _168_Lucene50_0.tim, _168_Lucene50_0.tip, _169.fdt, _169.fdx, _169.fnm, _169.nvd, _169.nvm, _169.si, _169_Lucene50_0.doc, _169_Lucene50_0.tim, _169_Lucene50_0.tip, _16a.fdt, _16a.fdx, _16a.fnm, _16a.nvd, _16a.nvm, _16a.si, _16a_Lucene50_0.doc, _16a_Lucene50_0.tim, _16a_Lucene50_0.tip, _16b.fdt, _16b.fdx, _16b.fnm, _16b.nvd, _16b.nvm, _16b.si, _16b_Lucene50_0.doc, _16b_Lucene50_0.tim, _16b_Lucene50_0.tip, _16c.fdt, _16c.fdx, _16c.fnm, _16c.nvd, _16c.nvm, _16c.si, _16c_Lucene50_0.doc, _16c_Lucene50_0.tim, _16c_Lucene50_0.tip, _16d.fdt, _16d.fdx, _16d.fnm, _16d.nvd, _16d.nvm, _16d.si, _16d_Lucene50_0.doc, _16d_Lucene50_0.tim, _16d_Lucene50_0.tip, _16e.fdt, _16e.fdx, _16e.fnm, _16e.nvd, _16e.nvm, _16e.si, _16e_Lucene50_0.doc, _16e_Lucene50_0.tim, _16e_Lucene50_0.tip, _16f.fdt, _16f.fdx, _16f.fnm, _16f.nvd, _16f.nvm, _16f.si, _16f_Lucene50_0.doc, _16f_Lucene50_0.tim, _16f_Lucene50_0.tip, _16g.fdt, _16g.fdx, _16g.fnm, _16g.nvd, _16g.nvm, _16g.si, _16g_Lucene50_0.doc, _16g_Lucene50_0.tim, _16g_Lucene50_0.tip, _16h.fdt, _16h.fdx, _16h.fnm, _16h.nvd, _16h.nvm, _16h.si, _16h_Lucene50_0.doc, _16h_Lucene50_0.tim, _16h_Lucene50_0.tip, _16i.fdt, _16i.fdx, _16i.fnm, _16i.nvd, _16i.nvm, _16i.si, _16i_Lucene50_0.doc, _16i_Lucene50_0.tim, _16i_Lucene50_0.tip, _16j.fdt, _16j.fdx, _16j.fnm, _16j.nvd, _16j.nvm, _16j.si, _16j_Lucene50_0.doc, _16j_Lucene50_0.tim, _16j_Lucene50_0.tip, segments_2]}]> but was:<[{indexVersion=1521590816599,generation=2,filelist=[_15f.fdt, _15f.fdx, _15f.fnm, _15f.nvd, _15f.nvm, _15f.si, _15f_Lucene50_0.doc, _15f_Lucene50_0.tim, _15f_Lucene50_0.tip, _15g.fdt, _15g.fdx, _15g.fnm, _15g.nvd, _15g.nvm, _15g.si, _15g_Lucene50_0.doc, _15g_Lucene50_0.tim, _15g_Lucene50_0.tip, _15h.fdt, _15h.fdx, _15h.fnm, _15h.nvd, _15h.nvm, _15h.si, _15h_Lucene50_0.doc, _15h_Lucene50_0.tim, _15h_Lucene50_0.tip, _15i.fdt, _15i.fdx, _15i.fnm, _15i.nvd, _15i.nvm, _15i.si, _15i_Lucene50_0.doc, _15i_Lucene50_0.tim, _15i_Lucene50_0.tip, _15j.fdt, _15j.fdx, _15j.fnm, _15j.nvd, _15j.nvm, _15j.si, _15j_Lucene50_0.doc, _15j_Lucene50_0.tim, _15j_Lucene50_0.tip, _15k.fdt, _15k.fdx, _15k.fnm, _15k.nvd, _15k.nvm, _15k.si, _15k_Lucene50_0.doc, _15k_Lucene50_0.tim, _15k_Lucene50_0.tip, _15l.fdt, _15l.fdx, _15l.fnm, _15l.nvd, _15l.nvm, _15l.si, _15l_Lucene50_0.doc, _15l_Lucene50_0.tim, _15l_Lucene50_0.tip, _15m.fdt, _15m.fdx, _15m.fnm, _15m.nvd, _15m.nvm, _15m.si, _15m_Lucene50_0.doc, _15m_Lucene50_0.tim, _15m_Lucene50_0.tip, _15n.fdt, _15n.fdx, _15n.fnm, _15n.nvd, _15n.nvm, _15n.si, _15n_Lucene50_0.doc, _15n_Lucene50_0.tim, _15n_Lucene50_0.tip, _15o.fdt, _15o.fdx, _15o.fnm, _15o.nvd, _15o.nvm, _15o.si, _15o_Lucene50_0.doc, _15o_Lucene50_0.tim, _15o_Lucene50_0.tip, _15p.fdt, _15p.fdx, _15p.fnm, _15p.nvd, _15p.nvm, _15p.si, _15p_Lucene50_0.doc, _15p_Lucene50_0.tim, _15p_Lucene50_0.tip, _15q.fdt, _15q.fdx, _15q.fnm, _15q.nvd, _15q.nvm, _15q.si, _15q_Lucene50_0.doc, _15q_Lucene50_0.tim, _15q_Lucene50_0.tip, _15r.fdt, _15r.fdx, _15r.fnm, _15r.nvd, _15r.nvm, _15r.si, _15r_Lucene50_0.doc, _15r_Lucene50_0.tim, _15r_Lucene50_0.tip, _15s.fdt, _15s.fdx, _15s.fnm, _15s.nvd, _15s.nvm, _15s.si, _15s_Lucene50_0.doc, _15s_Lucene50_0.tim, _15s_Lucene50_0.tip, _15t.fdt, _15t.fdx, _15t.fnm, _15t.nvd, _15t.nvm, _15t.si, _15t_Lucene50_0.doc, _15t_Lucene50_0.tim, _15t_Lucene50_0.tip, _15u.fdt, _15u.fdx, _15u.fnm, _15u.nvd, _15u.nvm, _15u.si, _15u_Lucene50_0.doc, _15u_Lucene50_0.tim, _15u_Lucene50_0.tip, _15v.fdt, _15v.fdx, _15v.fnm, _15v.nvd, _15v.nvm, _15v.si, _15v_Lucene50_0.doc, _15v_Lucene50_0.tim, _15v_Lucene50_0.tip, _15w.fdt, _15w.fdx, _15w.fnm, _15w.nvd, _15w.nvm, _15w.si, _15w_Lucene50_0.doc, _15w_Lucene50_0.tim, _15w_Lucene50_0.tip, _15x.fdt, _15x.fdx, _15x.fnm, _15x.nvd, _15x.nvm, _15x.si, _15x_Lucene50_0.doc, _15x_Lucene50_0.tim, _15x_Lucene50_0.tip, _15y.fdt, _15y.fdx, _15y.fnm, _15y.nvd, _15y.nvm, _15y.si, _15y_Lucene50_0.doc, _15y_Lucene50_0.tim, _15y_Lucene50_0.tip, _15z.fdt, _15z.fdx, _15z.fnm, _15z.nvd, _15z.nvm, _15z.si, _15z_Lucene50_0.doc, _15z_Lucene50_0.tim, _15z_Lucene50_0.tip, _161.fdt, _161.fdx, _161.fnm, _161.nvd, _161.nvm, _161.si, _161_Lucene50_0.doc, _161_Lucene50_0.tim, _161_Lucene50_0.tip, _162.fdt, _162.fdx, _162.fnm, _162.nvd, _162.nvm, _162.si, _162_Lucene50_0.doc, _162_Lucene50_0.tim, _162_Lucene50_0.tip, _163.fdt, _163.fdx, _163.fnm, _163.nvd, _163.nvm, _163.si, _163_Lucene50_0.doc, _163_Lucene50_0.tim, _163_Lucene50_0.tip, _164.fdt, _164.fdx, _164.fnm, _164.nvd, _164.nvm, _164.si, _164_Lucene50_0.doc, _164_Lucene50_0.tim, _164_Lucene50_0.tip, _165.fdt, _165.fdx, _165.fnm, _165.nvd, _165.nvm, _165.si, _165_Lucene50_0.doc, _165_Lucene50_0.tim, _165_Lucene50_0.tip, _166.fdt, _166.fdx, _166.fnm, _166.nvd, _166.nvm, _166.si, _166_Lucene50_0.doc, _166_Lucene50_0.tim, _166_Lucene50_0.tip, _167.fdt, _167.fdx, _167.fnm, _167.nvd, _167.nvm, _167.si, _167_Lucene50_0.doc, _167_Lucene50_0.tim, _167_Lucene50_0.tip, _168.fdt, _168.fdx, _168.fnm, _168.nvd, _168.nvm, _168.si, _168_Lucene50_0.doc, _168_Lucene50_0.tim, _168_Lucene50_0.tip, _169.fdt, _169.fdx, _169.fnm, _169.nvd, _169.nvm, _169.si, _169_Lucene50_0.doc, _169_Lucene50_0.tim, _169_Lucene50_0.tip, _16a.fdt, _16a.fdx, _16a.fnm, _16a.nvd, _16a.nvm, _16a.si, _16a_Lucene50_0.doc, _16a_Lucene50_0.tim, _16a_Lucene50_0.tip, _16b.fdt, _16b.fdx, _16b.fnm, _16b.nvd, _16b.nvm, _16b.si, _16b_Lucene50_0.doc, _16b_Lucene50_0.tim, _16b_Lucene50_0.tip, _16c.fdt, _16c.fdx, _16c.fnm, _16c.nvd, _16c.nvm, _16c.si, _16c_Lucene50_0.doc, _16c_Lucene50_0.tim, _16c_Lucene50_0.tip, _16d.fdt, _16d.fdx, _16d.fnm, _16d.nvd, _16d.nvm, _16d.si, _16d_Lucene50_0.doc, _16d_Lucene50_0.tim, _16d_Lucene50_0.tip, _16e.fdt, _16e.fdx, _16e.fnm, _16e.nvd, _16e.nvm, _16e.si, _16e_Lucene50_0.doc, _16e_Lucene50_0.tim, _16e_Lucene50_0.tip, _16f.fdt, _16f.fdx, _16f.fnm, _16f.nvd, _16f.nvm, _16f.si, _16f_Lucene50_0.doc, _16f_Lucene50_0.tim, _16f_Lucene50_0.tip, _16g.fdt, _16g.fdx, _16g.fnm, _16g.nvd, _16g.nvm, _16g.si, _16g_Lucene50_0.doc, _16g_Lucene50_0.tim, _16g_Lucene50_0.tip, _16h.fdt, _16h.fdx, _16h.fnm, _16h.nvd, _16h.nvm, _16h.si, _16h_Lucene50_0.doc, _16h_Lucene50_0.tim, _16h_Lucene50_0.tip, _16i.fdt, _16i.fdx, _16i.fnm, _16i.nvd, _16i.nvm, _16i.si, _16i_Lucene50_0.doc, _16i_Lucene50_0.tim, _16i_Lucene50_0.tip, _16j.fdt, _16j.fdx, _16j.fnm, _16j.nvd, _16j.nvm, _16j.si, _16j_Lucene50_0.doc, _16j_Lucene50_0.tim, _16j_Lucene50_0.tip, segments_2]}, {indexVersion=1521590816599,generation=3,filelist=[_160.fdt, _160.fdx, _160.fnm, _160.nvd, _160.nvm, _160.si, _160_Lucene50_0.doc, _160_Lucene50_0.tim, _160_Lucene50_0.tip, _161.fdt, _161.fdx, _161.fnm, _161.nvd, _161.nvm, _161.si, _161_Lucene50_0.doc, _161_Lucene50_0.tim, _161_Lucene50_0.tip, _162.fdt, _162.fdx, _162.fnm, _162.nvd, _162.nvm, _162.si, _162_Lucene50_0.doc, _162_Lucene50_0.tim, _162_Lucene50_0.tip, _163.fdt, _163.fdx, _163.fnm, _163.nvd, _163.nvm, _163.si, _163_Lucene50_0.doc, _163_Lucene50_0.tim, _163_Lucene50_0.tip, _164.fdt, _164.fdx, _164.fnm, _164.nvd, _164.nvm, _164.si, _164_Lucene50_0.doc, _164_Lucene50_0.tim, _164_Lucene50_0.tip, _165.fdt, _165.fdx, _165.fnm, _165.nvd, _165.nvm, _165.si, _165_Lucene50_0.doc, _165_Lucene50_0.tim, _165_Lucene50_0.tip, _166.fdt, _166.fdx, _166.fnm, _166.nvd, _166.nvm, _166.si, _166_Lucene50_0.doc, _166_Lucene50_0.tim, _166_Lucene50_0.tip, _167.fdt, _167.fdx, _167.fnm, _167.nvd, _167.nvm, _167.si, _167_Lucene50_0.doc, _167_Lucene50_0.tim, _167_Lucene50_0.tip, _168.fdt, _168.fdx, _168.fnm, _168.nvd, _168.nvm, _168.si, _168_Lucene50_0.doc, _168_Lucene50_0.tim, _168_Lucene50_0.tip, _169.fdt, _169.fdx, _169.fnm, _169.nvd, _169.nvm, _169.si, _169_Lucene50_0.doc, _169_Lucene50_0.tim, _169_Lucene50_0.tip, _16a.fdt, _16a.fdx, _16a.fnm, _16a.nvd, _16a.nvm, _16a.si, _16a_Lucene50_0.doc, _16a_Lucene50_0.tim, _16a_Lucene50_0.tip, _16b.fdt, _16b.fdx, _16b.fnm, _16b.nvd, _16b.nvm, _16b.si, _16b_Lucene50_0.doc, _16b_Lucene50_0.tim, _16b_Lucene50_0.tip, _16c.fdt, _16c.fdx, _16c.fnm, _16c.nvd, _16c.nvm, _16c.si, _16c_Lucene50_0.doc, _16c_Lucene50_0.tim, _16c_Lucene50_0.tip, _16d.fdt, _16d.fdx, _16d.fnm, _16d.nvd, _16d.nvm, _16d.si, _16d_Lucene50_0.doc, _16d_Lucene50_0.tim, _16d_Lucene50_0.tip, _16e.fdt, _16e.fdx, _16e.fnm, _16e.nvd, _16e.nvm, _16e.si, _16e_Lucene50_0.doc, _16e_Lucene50_0.tim, _16e_Lucene50_0.tip, _16f.fdt, _16f.fdx, _16f.fnm, _16f.nvd, _16f.nvm, _16f.si, _16f_Lucene50_0.doc, _16f_Lucene50_0.tim, _16f_Lucene50_0.tip, _16g.fdt, _16g.fdx, _16g.fnm, _16g.nvd, _16g.nvm, _16g.si, _16g_Lucene50_0.doc, _16g_Lucene50_0.tim, _16g_Lucene50_0.tip, _16h.fdt, _16h.fdx, _16h.fnm, _16h.nvd, _16h.nvm, _16h.si, _16h_Lucene50_0.doc, _16h_Lucene50_0.tim, _16h_Lucene50_0.tip, _16i.fdt, _16i.fdx, _16i.fnm, _16i.nvd, _16i.nvm, _16i.si, _16i_Lucene50_0.doc, _16i_Lucene50_0.tim, _16i_Lucene50_0.tip, _16j.fdt, _16j.fdx, _16j.fnm, _16j.nvd, _16j.nvm, _16j.si, _16j_Lucene50_0.doc, _16j_Lucene50_0.tim, _16j_Lucene50_0.tip, segments_3]}]>

Stack Trace:
java.lang.AssertionError: expected:<[{indexVersion=1521590816599,generation=2,filelist=[_15f.fdt, _15f.fdx, _15f.fnm, _15f.nvd, _15f.nvm, _15f.si, _15f_Lucene50_0.doc, _15f_Lucene50_0.tim, _15f_Lucene50_0.tip, _15g.fdt, _15g.fdx, _15g.fnm, _15g.nvd, _15g.nvm, _15g.si, _15g_Lucene50_0.doc, _15g_Lucene50_0.tim, _15g_Lucene50_0.tip, _15h.fdt, _15h.fdx, _15h.fnm, _15h.nvd, _15h.nvm, _15h.si, _15h_Lucene50_0.doc, _15h_Lucene50_0.tim, _15h_Lucene50_0.tip, _15i.fdt, _15i.fdx, _15i.fnm, _15i.nvd, _15i.nvm, _15i.si, _15i_Lucene50_0.doc, _15i_Lucene50_0.tim, _15i_Lucene50_0.tip, _15j.fdt, _15j.fdx, _15j.fnm, _15j.nvd, _15j.nvm, _15j.si, _15j_Lucene50_0.doc, _15j_Lucene50_0.tim, _15j_Lucene50_0.tip, _15k.fdt, _15k.fdx, _15k.fnm, _15k.nvd, _15k.nvm, _15k.si, _15k_Lucene50_0.doc, _15k_Lucene50_0.tim, _15k_Lucene50_0.tip, _15l.fdt, _15l.fdx, _15l.fnm, _15l.nvd, _15l.nvm, _15l.si, _15l_Lucene50_0.doc, _15l_Lucene50_0.tim, _15l_Lucene50_0.tip, _15m.fdt, _15m.fdx, _15m.fnm, _15m.nvd, _15m.nvm, _15m.si, _15m_Lucene50_0.doc, _15m_Lucene50_0.tim, _15m_Lucene50_0.tip, _15n.fdt, _15n.fdx, _15n.fnm, _15n.nvd, _15n.nvm, _15n.si, _15n_Lucene50_0.doc, _15n_Lucene50_0.tim, _15n_Lucene50_0.tip, _15o.fdt, _15o.fdx, _15o.fnm, _15o.nvd, _15o.nvm, _15o.si, _15o_Lucene50_0.doc, _15o_Lucene50_0.tim, _15o_Lucene50_0.tip, _15p.fdt, _15p.fdx, _15p.fnm, _15p.nvd, _15p.nvm, _15p.si, _15p_Lucene50_0.doc, _15p_Lucene50_0.tim, _15p_Lucene50_0.tip, _15q.fdt, _15q.fdx, _15q.fnm, _15q.nvd, _15q.nvm, _15q.si, _15q_Lucene50_0.doc, _15q_Lucene50_0.tim, _15q_Lucene50_0.tip, _15r.fdt, _15r.fdx, _15r.fnm, _15r.nvd, _15r.nvm, _15r.si, _15r_Lucene50_0.doc, _15r_Lucene50_0.tim, _15r_Lucene50_0.tip, _15s.fdt, _15s.fdx, _15s.fnm, _15s.nvd, _15s.nvm, _15s.si, _15s_Lucene50_0.doc, _15s_Lucene50_0.tim, _15s_Lucene50_0.tip, _15t.fdt, _15t.fdx, _15t.fnm, _15t.nvd, _15t.nvm, _15t.si, _15t_Lucene50_0.doc, _15t_Lucene50_0.tim, _15t_Lucene50_0.tip, _15u.fdt, _15u.fdx, _15u.fnm, _15u.nvd, _15u.nvm, _15u.si, _15u_Lucene50_0.doc, _15u_Lucene50_0.tim, _15u_Lucene50_0.tip, _15v.fdt, _15v.fdx, _15v.fnm, _15v.nvd, _15v.nvm, _15v.si, _15v_Lucene50_0.doc, _15v_Lucene50_0.tim, _15v_Lucene50_0.tip, _15w.fdt, _15w.fdx, _15w.fnm, _15w.nvd, _15w.nvm, _15w.si, _15w_Lucene50_0.doc, _15w_Lucene50_0.tim, _15w_Lucene50_0.tip, _15x.fdt, _15x.fdx, _15x.fnm, _15x.nvd, _15x.nvm, _15x.si, _15x_Lucene50_0.doc, _15x_Lucene50_0.tim, _15x_Lucene50_0.tip, _15y.fdt, _15y.fdx, _15y.fnm, _15y.nvd, _15y.nvm, _15y.si, _15y_Lucene50_0.doc, _15y_Lucene50_0.tim, _15y_Lucene50_0.tip, _15z.fdt, _15z.fdx, _15z.fnm, _15z.nvd, _15z.nvm, _15z.si, _15z_Lucene50_0.doc, _15z_Lucene50_0.tim, _15z_Lucene50_0.tip, _161.fdt, _161.fdx, _161.fnm, _161.nvd, _161.nvm, _161.si, _161_Lucene50_0.doc, _161_Lucene50_0.tim, _161_Lucene50_0.tip, _162.fdt, _162.fdx, _162.fnm, _162.nvd, _162.nvm, _162.si, _162_Lucene50_0.doc, _162_Lucene50_0.tim, _162_Lucene50_0.tip, _163.fdt, _163.fdx, _163.fnm, _163.nvd, _163.nvm, _163.si, _163_Lucene50_0.doc, _163_Lucene50_0.tim, _163_Lucene50_0.tip, _164.fdt, _164.fdx, _164.fnm, _164.nvd, _164.nvm, _164.si, _164_Lucene50_0.doc, _164_Lucene50_0.tim, _164_Lucene50_0.tip, _165.fdt, _165.fdx, _165.fnm, _165.nvd, _165.nvm, _165.si, _165_Lucene50_0.doc, _165_Lucene50_0.tim, _165_Lucene50_0.tip, _166.fdt, _166.fdx, _166.fnm, _166.nvd, _166.nvm, _166.si, _166_Lucene50_0.doc, _166_Lucene50_0.tim, _166_Lucene50_0.tip, _167.fdt, _167.fdx, _167.fnm, _167.nvd, _167.nvm, _167.si, _167_Lucene50_0.doc, _167_Lucene50_0.tim, _167_Lucene50_0.tip, _168.fdt, _168.fdx, _168.fnm, _168.nvd, _168.nvm, _168.si, _168_Lucene50_0.doc, _168_Lucene50_0.tim, _168_Lucene50_0.tip, _169.fdt, _169.fdx, _169.fnm, _169.nvd, _169.nvm, _169.si, _169_Lucene50_0.doc, _169_Lucene50_0.tim, _169_Lucene50_0.tip, _16a.fdt, _16a.fdx, _16a.fnm, _16a.nvd, _16a.nvm, _16a.si, _16a_Lucene50_0.doc, _16a_Lucene50_0.tim, _16a_Lucene50_0.tip, _16b.fdt, _16b.fdx, _16b.fnm, _16b.nvd, _16b.nvm, _16b.si, _16b_Lucene50_0.doc, _16b_Lucene50_0.tim, _16b_Lucene50_0.tip, _16c.fdt, _16c.fdx, _16c.fnm, _16c.nvd, _16c.nvm, _16c.si, _16c_Lucene50_0.doc, _16c_Lucene50_0.tim, _16c_Lucene50_0.tip, _16d.fdt, _16d.fdx, _16d.fnm, _16d.nvd, _16d.nvm, _16d.si, _16d_Lucene50_0.doc, _16d_Lucene50_0.tim, _16d_Lucene50_0.tip, _16e.fdt, _16e.fdx, _16e.fnm, _16e.nvd, _16e.nvm, _16e.si, _16e_Lucene50_0.doc, _16e_Lucene50_0.tim, _16e_Lucene50_0.tip, _16f.fdt, _16f.fdx, _16f.fnm, _16f.nvd, _16f.nvm, _16f.si, _16f_Lucene50_0.doc, _16f_Lucene50_0.tim, _16f_Lucene50_0.tip, _16g.fdt, _16g.fdx, _16g.fnm, _16g.nvd, _16g.nvm, _16g.si, _16g_Lucene50_0.doc, _16g_Lucene50_0.tim, _16g_Lucene50_0.tip, _16h.fdt, _16h.fdx, _16h.fnm, _16h.nvd, _16h.nvm, _16h.si, _16h_Lucene50_0.doc, _16h_Lucene50_0.tim, _16h_Lucene50_0.tip, _16i.fdt, _16i.fdx, _16i.fnm, _16i.nvd, _16i.nvm, _16i.si, _16i_Lucene50_0.doc, _16i_Lucene50_0.tim, _16i_Lucene50_0.tip, _16j.fdt, _16j.fdx, _16j.fnm, _16j.nvd, _16j.nvm, _16j.si, _16j_Lucene50_0.doc, _16j_Lucene50_0.tim, _16j_Lucene50_0.tip, segments_2]}]> but was:<[{indexVersion=1521590816599,generation=2,filelist=[_15f.fdt, _15f.fdx, _15f.fnm, _15f.nvd, _15f.nvm, _15f.si, _15f_Lucene50_0.doc, _15f_Lucene50_0.tim, _15f_Lucene50_0.tip, _15g.fdt, _15g.fdx, _15g.fnm, _15g.nvd, _15g.nvm, _15g.si, _15g_Lucene50_0.doc, _15g_Lucene50_0.tim, _15g_Lucene50_0.tip, _15h.fdt, _15h.fdx, _15h.fnm, _15h.nvd, _15h.nvm, _15h.si, _15h_Lucene50_0.doc, _15h_Lucene50_0.tim, _15h_Lucene50_0.tip, _15i.fdt, _15i.fdx, _15i.fnm, _15i.nvd, _15i.nvm, _15i.si, _15i_Lucene50_0.doc, _15i_Lucene50_0.tim, _15i_Lucene50_0.tip, _15j.fdt, _15j.fdx, _15j.fnm, _15j.nvd, _15j.nvm, _15j.si, _15j_Lucene50_0.doc, _15j_Lucene50_0.tim, _15j_Lucene50_0.tip, _15k.fdt, _15k.fdx, _15k.fnm, _15k.nvd, _15k.nvm, _15k.si, _15k_Lucene50_0.doc, _15k_Lucene50_0.tim, _15k_Lucene50_0.tip, _15l.fdt, _15l.fdx, _15l.fnm, _15l.nvd, _15l.nvm, _15l.si, _15l_Lucene50_0.doc, _15l_Lucene50_0.tim, _15l_Lucene50_0.tip, _15m.fdt, _15m.fdx, _15m.fnm, _15m.nvd, _15m.nvm, _15m.si, _15m_Lucene50_0.doc, _15m_Lucene50_0.tim, _15m_Lucene50_0.tip, _15n.fdt, _15n.fdx, _15n.fnm, _15n.nvd, _15n.nvm, _15n.si, _15n_Lucene50_0.doc, _15n_Lucene50_0.tim, _15n_Lucene50_0.tip, _15o.fdt, _15o.fdx, _15o.fnm, _15o.nvd, _15o.nvm, _15o.si, _15o_Lucene50_0.doc, _15o_Lucene50_0.tim, _15o_Lucene50_0.tip, _15p.fdt, _15p.fdx, _15p.fnm, _15p.nvd, _15p.nvm, _15p.si, _15p_Lucene50_0.doc, _15p_Lucene50_0.tim, _15p_Lucene50_0.tip, _15q.fdt, _15q.fdx, _15q.fnm, _15q.nvd, _15q.nvm, _15q.si, _15q_Lucene50_0.doc, _15q_Lucene50_0.tim, _15q_Lucene50_0.tip, _15r.fdt, _15r.fdx, _15r.fnm, _15r.nvd, _15r.nvm, _15r.si, _15r_Lucene50_0.doc, _15r_Lucene50_0.tim, _15r_Lucene50_0.tip, _15s.fdt, _15s.fdx, _15s.fnm, _15s.nvd, _15s.nvm, _15s.si, _15s_Lucene50_0.doc, _15s_Lucene50_0.tim, _15s_Lucene50_0.tip, _15t.fdt, _15t.fdx, _15t.fnm, _15t.nvd, _15t.nvm, _15t.si, _15t_Lucene50_0.doc, _15t_Lucene50_0.tim, _15t_Lucene50_0.tip, _15u.fdt, _15u.fdx, _15u.fnm, _15u.nvd, _15u.nvm, _15u.si, _15u_Lucene50_0.doc, _15u_Lucene50_0.tim, _15u_Lucene50_0.tip, _15v.fdt, _15v.fdx, _15v.fnm, _15v.nvd, _15v.nvm, _15v.si, _15v_Lucene50_0.doc, _15v_Lucene50_0.tim, _15v_Lucene50_0.tip, _15w.fdt, _15w.fdx, _15w.fnm, _15w.nvd, _15w.nvm, _15w.si, _15w_Lucene50_0.doc, _15w_Lucene50_0.tim, _15w_Lucene50_0.tip, _15x.fdt, _15x.fdx, _15x.fnm, _15x.nvd, _15x.nvm, _15x.si, _15x_Lucene50_0.doc, _15x_Lucene50_0.tim, _15x_Lucene50_0.tip, _15y.fdt, _15y.fdx, _15y.fnm, _15y.nvd, _15y.nvm, _15y.si, _15y_Lucene50_0.doc, _15y_Lucene50_0.tim, _15y_Lucene50_0.tip, _15z.fdt, _15z.fdx, _15z.fnm, _15z.nvd, _15z.nvm, _15z.si, _15z_Lucene50_0.doc, _15z_Lucene50_0.tim, _15z_Lucene50_0.tip, _161.fdt, _161.fdx, _161.fnm, _161.nvd, _161.nvm, _161.si, _161_Lucene50_0.doc, _161_Lucene50_0.tim, _161_Lucene50_0.tip, _162.fdt, _162.fdx, _162.fnm, _162.nvd, _162.nvm, _162.si, _162_Lucene50_0.doc, _162_Lucene50_0.tim, _162_Lucene50_0.tip, _163.fdt, _163.fdx, _163.fnm, _163.nvd, _163.nvm, _163.si, _163_Lucene50_0.doc, _163_Lucene50_0.tim, _163_Lucene50_0.tip, _164.fdt, _164.fdx, _164.fnm, _164.nvd, _164.nvm, _164.si, _164_Lucene50_0.doc, _164_Lucene50_0.tim, _164_Lucene50_0.tip, _165.fdt, _165.fdx, _165.fnm, _165.nvd, _165.nvm, _165.si, _165_Lucene50_0.doc, _165_Lucene50_0.tim, _165_Lucene50_0.tip, _166.fdt, _166.fdx, _166.fnm, _166.nvd, _166.nvm, _166.si, _166_Lucene50_0.doc, _166_Lucene50_0.tim, _166_Lucene50_0.tip, _167.fdt, _167.fdx, _167.fnm, _167.nvd, _167.nvm, _167.si, _167_Lucene50_0.doc, _167_Lucene50_0.tim, _167_Lucene50_0.tip, _168.fdt, _168.fdx, _168.fnm, _168.nvd, _168.nvm, _168.si, _168_Lucene50_0.doc, _168_Lucene50_0.tim, _168_Lucene50_0.tip, _169.fdt, _169.fdx, _169.fnm, _169.nvd, _169.nvm, _169.si, _169_Lucene50_0.doc, _169_Lucene50_0.tim, _169_Lucene50_0.tip, _16a.fdt, _16a.fdx, _16a.fnm, _16a.nvd, _16a.nvm, _16a.si, _16a_Lucene50_0.doc, _16a_Lucene50_0.tim, _16a_Lucene50_0.tip, _16b.fdt, _16b.fdx, _16b.fnm, _16b.nvd, _16b.nvm, _16b.si, _16b_Lucene50_0.doc, _16b_Lucene50_0.tim, _16b_Lucene50_0.tip, _16c.fdt, _16c.fdx, _16c.fnm, _16c.nvd, _16c.nvm, _16c.si, _16c_Lucene50_0.doc, _16c_Lucene50_0.tim, _16c_Lucene50_0.tip, _16d.fdt, _16d.fdx, _16d.fnm, _16d.nvd, _16d.nvm, _16d.si, _16d_Lucene50_0.doc, _16d_Lucene50_0.tim, _16d_Lucene50_0.tip, _16e.fdt, _16e.fdx, _16e.fnm, _16e.nvd, _16e.nvm, _16e.si, _16e_Lucene50_0.doc, _16e_Lucene50_0.tim, _16e_Lucene50_0.tip, _16f.fdt, _16f.fdx, _16f.fnm, _16f.nvd, _16f.nvm, _16f.si, _16f_Lucene50_0.doc, _16f_Lucene50_0.tim, _16f_Lucene50_0.tip, _16g.fdt, _16g.fdx, _16g.fnm, _16g.nvd, _16g.nvm, _16g.si, _16g_Lucene50_0.doc, _16g_Lucene50_0.tim, _16g_Lucene50_0.tip, _16h.fdt, _16h.fdx, _16h.fnm, _16h.nvd, _16h.nvm, _16h.si, _16h_Lucene50_0.doc, _16h_Lucene50_0.tim, _16h_Lucene50_0.tip, _16i.fdt, _16i.fdx, _16i.fnm, _16i.nvd, _16i.nvm, _16i.si, _16i_Lucene50_0.doc, _16i_Lucene50_0.tim, _16i_Lucene50_0.tip, _16j.fdt, _16j.fdx, _16j.fnm, _16j.nvd, _16j.nvm, _16j.si, _16j_Lucene50_0.doc, _16j_Lucene50_0.tim, _16j_Lucene50_0.tip, segments_2]}, {indexVersion=1521590816599,generation=3,filelist=[_160.fdt, _160.fdx, _160.fnm, _160.nvd, _160.nvm, _160.si, _160_Lucene50_0.doc, _160_Lucene50_0.tim, _160_Lucene50_0.tip, _161.fdt, _161.fdx, _161.fnm, _161.nvd, _161.nvm, _161.si, _161_Lucene50_0.doc, _161_Lucene50_0.tim, _161_Lucene50_0.tip, _162.fdt, _162.fdx, _162.fnm, _162.nvd, _162.nvm, _162.si, _162_Lucene50_0.doc, _162_Lucene50_0.tim, _162_Lucene50_0.tip, _163.fdt, _163.fdx, _163.fnm, _163.nvd, _163.nvm, _163.si, _163_Lucene50_0.doc, _163_Lucene50_0.tim, _163_Lucene50_0.tip, _164.fdt, _164.fdx, _164.fnm, _164.nvd, _164.nvm, _164.si, _164_Lucene50_0.doc, _164_Lucene50_0.tim, _164_Lucene50_0.tip, _165.fdt, _165.fdx, _165.fnm, _165.nvd, _165.nvm, _165.si, _165_Lucene50_0.doc, _165_Lucene50_0.tim, _165_Lucene50_0.tip, _166.fdt, _166.fdx, _166.fnm, _166.nvd, _166.nvm, _166.si, _166_Lucene50_0.doc, _166_Lucene50_0.tim, _166_Lucene50_0.tip, _167.fdt, _167.fdx, _167.fnm, _167.nvd, _167.nvm, _167.si, _167_Lucene50_0.doc, _167_Lucene50_0.tim, _167_Lucene50_0.tip, _168.fdt, _168.fdx, _168.fnm, _168.nvd, _168.nvm, _168.si, _168_Lucene50_0.doc, _168_Lucene50_0.tim, _168_Lucene50_0.tip, _169.fdt, _169.fdx, _169.fnm, _169.nvd, _169.nvm, _169.si, _169_Lucene50_0.doc, _169_Lucene50_0.tim, _169_Lucene50_0.tip, _16a.fdt, _16a.fdx, _16a.fnm, _16a.nvd, _16a.nvm, _16a.si, _16a_Lucene50_0.doc, _16a_Lucene50_0.tim, _16a_Lucene50_0.tip, _16b.fdt, _16b.fdx, _16b.fnm, _16b.nvd, _16b.nvm, _16b.si, _16b_Lucene50_0.doc, _16b_Lucene50_0.tim, _16b_Lucene50_0.tip, _16c.fdt, _16c.fdx, _16c.fnm, _16c.nvd, _16c.nvm, _16c.si, _16c_Lucene50_0.doc, _16c_Lucene50_0.tim, _16c_Lucene50_0.tip, _16d.fdt, _16d.fdx, _16d.fnm, _16d.nvd, _16d.nvm, _16d.si, _16d_Lucene50_0.doc, _16d_Lucene50_0.tim, _16d_Lucene50_0.tip, _16e.fdt, _16e.fdx, _16e.fnm, _16e.nvd, _16e.nvm, _16e.si, _16e_Lucene50_0.doc, _16e_Lucene50_0.tim, _16e_Lucene50_0.tip, _16f.fdt, _16f.fdx, _16f.fnm, _16f.nvd, _16f.nvm, _16f.si, _16f_Lucene50_0.doc, _16f_Lucene50_0.tim, _16f_Lucene50_0.tip, _16g.fdt, _16g.fdx, _16g.fnm, _16g.nvd, _16g.nvm, _16g.si, _16g_Lucene50_0.doc, _16g_Lucene50_0.tim, _16g_Lucene50_0.tip, _16h.fdt, _16h.fdx, _16h.fnm, _16h.nvd, _16h.nvm, _16h.si, _16h_Lucene50_0.doc, _16h_Lucene50_0.tim, _16h_Lucene50_0.tip, _16i.fdt, _16i.fdx, _16i.fnm, _16i.nvd, _16i.nvm, _16i.si, _16i_Lucene50_0.doc, _16i_Lucene50_0.tim, _16i_Lucene50_0.tip, _16j.fdt, _16j.fdx, _16j.fnm, _16j.nvd, _16j.nvm, _16j.si, _16j_Lucene50_0.doc, _16j_Lucene50_0.tim, _16j_Lucene50_0.tip, segments_3]}]>
	at __randomizedtesting.SeedInfo.seed([8C1E58940135899E:A9C943A4717D879D]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:147)
	at org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload(TestReplicationHandler.java:1284)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.handler.admin.AutoscalingHistoryHandlerTest.testHistory

Error Message:
expected:<8> but was:<10>

Stack Trace:
java.lang.AssertionError: expected:<8> but was:<10>
	at __randomizedtesting.SeedInfo.seed([8C1E58940135899E:E1E2FC69BB7D7699]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.handler.admin.AutoscalingHistoryHandlerTest.testHistory(AutoscalingHistoryHandlerTest.java:354)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates.test_dv

Error Message:
java.lang.RuntimeException: Could not load collection from ZK: test_col

Stack Trace:
java.util.concurrent.ExecutionException: java.lang.RuntimeException: Could not load collection from ZK: test_col
	at __randomizedtesting.SeedInfo.seed([8C1E58940135899E:BA0A3AD28B68B38F]:0)
	at java.util.concurrent.FutureTask.report(FutureTask.java:122)
	at java.util.concurrent.FutureTask.get(FutureTask.java:192)
	at org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates.checkField(TestStressCloudBlindAtomicUpdates.java:288)
	at org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates.test_dv(TestStressCloudBlindAtomicUpdates.java:198)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.RuntimeException: Could not load collection from ZK: test_col
	at org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates$Worker.run(TestStressCloudBlindAtomicUpdates.java:416)
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	... 1 more
Caused by: org.apache.solr.common.SolrException: Could not load collection from ZK: test_col
	at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1236)
	at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:674)
	at org.apache.solr.common.cloud.ClusterState$CollectionRef.get(ClusterState.java:386)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.getDocCollection(CloudSolrClient.java:1207)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:850)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:818)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
	at org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates$Worker.doRandomAtomicUpdate(TestStressCloudBlindAtomicUpdates.java:375)
	at org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates$Worker.run(TestStressCloudBlindAtomicUpdates.java:404)
	... 6 more
Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/test_col/state.json
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
	at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
	at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
	at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
	at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1248)
	at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1234)
	... 15 more


FAILED:  org.apache.solr.cloud.hdfs.HdfsUnloadDistributedZkTest.test

Error Message:
Captured an uncaught exception in thread: Thread[id=2892, name=testExecutor-1036-thread-18, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=2892, name=testExecutor-1036-thread-18, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest]
Caused by: java.lang.RuntimeException: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:36627/p_cib/er
	at __randomizedtesting.SeedInfo.seed([8C1E58940135899E]:0)
	at org.apache.solr.cloud.BasicDistributedZkTest.lambda$createCollectionInOneInstance$0(BasicDistributedZkTest.java:596)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:36627/p_cib/er
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:654)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
	at org.apache.solr.cloud.BasicDistributedZkTest.lambda$createCollectionInOneInstance$0(BasicDistributedZkTest.java:594)
	... 4 more
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:171)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:542)
	... 9 more




Build Log:
[...truncated 2029 lines...]
   [junit4] Suite: org.apache.lucene.index.TestIndexWriterOnVMError
   [junit4]   1> Unexpected exception: dumping fake-exception-log:...
   [junit4]   1> 
   [junit4]   1> TEST: got expected fake exc:Fake UnknownError
   [junit4]   1> java.lang.UnknownError: Fake UnknownError
   [junit4]   1> 	at org.apache.lucene.index.TestIndexWriterOnVMError$3.eval(TestIndexWriterOnVMError.java:270)
   [junit4]   1> 	at org.apache.lucene.store.MockDirectoryWrapper.maybeThrowDeterministicException(MockDirectoryWrapper.java:1022)
   [junit4]   1> 	at org.apache.lucene.store.MockIndexOutputWrapper.writeBytes(MockIndexOutputWrapper.java:144)
   [junit4]   1> 	at org.apache.lucene.store.MockIndexOutputWrapper.writeByte(MockIndexOutputWrapper.java:126)
   [junit4]   1> 	at org.apache.lucene.store.DataOutput.writeInt(DataOutput.java:73)
   [junit4]   1> 	at org.apache.lucene.store.DataOutput.writeLong(DataOutput.java:213)
   [junit4]   1> 	at org.apache.lucene.codecs.CodecUtil.writeCRC(CodecUtil.java:552)
   [junit4]   1> 	at org.apache.lucene.codecs.CodecUtil.writeFooter(CodecUtil.java:393)
   [junit4]   1> 	at org.apache.lucene.codecs.blockterms.BlockTermsWriter.close(BlockTermsWriter.java:183)
   [junit4]   1> 	at org.apache.lucene.util.IOUtils.close(IOUtils.java:89)
   [junit4]   1> 	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.close(PerFieldPostingsFormat.java:241)
   [junit4]   1> 	at org.apache.lucene.util.IOUtils.close(IOUtils.java:89)
   [junit4]   1> 	at org.apache.lucene.util.IOUtils.close(IOUtils.java:76)
   [junit4]   1> 	at org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:112)
   [junit4]   1> 	at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:163)
   [junit4]   1> 	at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:463)
   [junit4]   1> 	at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:556)
   [junit4]   1> 	at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:672)
   [junit4]   1> 	at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:454)
   [junit4]   1> 	at org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:103)
   [junit4]   1> 	at org.apache.lucene.index.TestIndexWriterOnVMError.doTest(TestIndexWriterOnVMError.java:175)
   [junit4]   1> 	at org.apache.lucene.index.TestIndexWriterOnVMError.testUnknownError(TestIndexWriterOnVMError.java:258)
   [junit4]   1> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   1> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   1> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   1> 	at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   1> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
   [junit4]   1> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
   [junit4]   1> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
   [junit4]   1> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
   [junit4]   1> 	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   1> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   1> 	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   1> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   1> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   1> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   1> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   1> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   1> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   1> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
   [junit4]   1> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
   [junit4]   1> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
   [junit4]   1> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
   [junit4]   1> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   1> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   1> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   1> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   1> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   1> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   1> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   1> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   1> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   1> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   1> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   1> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   1> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   1> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   1> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   1> 
   [junit4]   1> TEST: got expected fake exc:Fake UnknownError
   [junit4]   1> java.lang.UnknownError: Fake UnknownError
   [junit4]   1> 	at org.apache.lucene.index.TestIndexWriterOnVMError$3.eval(TestIndexWriterOnVMError.java:270)
   [junit4]   1> 	at org.apache.lucene.store.MockDirectoryWrapper.maybeThrowDeterministicException(MockDirectoryWrapper.java:1022)
   [junit4]   1> 	at org.apache.lucene.store.MockIndexOutputWrapper.writeBytes(MockIndexOutputWrapper.java:144)
   [junit4]   1> 	at org.apache.lucene.store.MockIndexOutputWrapper.writeByte(MockIndexOutputWrapper.java:126)
   [junit4]   1> 	at org.apache.lucene.store.DataOutput.writeInt(DataOutput.java:70)
   [junit4]   1> 	at org.apache.lucene.store.DataOutput.writeLong(DataOutput.java:213)
   [junit4]   1> 	at org.apache.lucene.codecs.lucene70.Lucene70DocValuesConsumer.addTermsDict(Lucene70DocValuesConsumer.java:480)
   [junit4]   1> 	at org.apache.lucene.codecs.lucene70.Lucene70DocValuesConsumer.doAddSortedField(Lucene70DocValuesConsumer.java:437)
   [junit4]   1> 	at org.apache.lucene.codecs.lucene70.Lucene70DocValuesConsumer.addSortedField(Lucene70DocValuesConsumer.java:394)
   [junit4]   1> 	at org.apache.lucene.codecs.asserting.AssertingDocValuesFormat$AssertingDocValuesConsumer.addSortedField(AssertingDocValuesFormat.java:141)
   [junit4]   1> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addSortedField(PerFieldDocValuesFormat.java:119)
   [junit4]   1> 	at org.apache.lucene.index.SortedDocValuesWriter.flush(SortedDocValuesWriter.java:163)
   [junit4]   1> 	at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:248)
   [junit4]   1> 	at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:132)
   [junit4]   1> 	at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:463)
   [junit4]   1> 	at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:556)
   [junit4]   1> 	at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:672)
   [junit4]   1> 	at org.apache.lucene.index.IndexWriter.prepareCommitInternal(IndexWriter.java:3337)
   [junit4]   1> 	at org.apache.lucene.index.IndexWriter.commitInternal(IndexWriter.java:3549)
   [junit4]   1> 	at org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3514)
   [junit4]   1> 	at org.apache.lucene.index.TestIndexWriterOnVMError.doTest(TestIndexWriterOnVMError.java:181)
   [junit4]   1> 	at org.apache.lucene.index.TestIndexWriterOnVMError.testUnknownError(TestIndexWriterOnVMError.java:258)
   [junit4]   1> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   1> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   1> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   1> 	at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   1> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
   [junit4]   1> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
   [junit4]   1> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
   [junit4]   1> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
   [junit4]   1> 	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   1> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   1> 	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   1> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   1> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   1> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   1> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   1> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   1> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   1> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
   [junit4]   1> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
   [junit4]   1> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
   [junit4]   1> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
   [junit4]   1> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   1> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   1> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   1> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   1> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   1> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   1> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   1> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   1> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   1> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   1> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   1> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   1> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   1> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   1> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   1> 
   [junit4]   1> TEST: got expected fake exc:Fake UnknownError
   [junit4]   1> java.lang.UnknownError: Fake UnknownError
   [junit4]   1> 	at org.apache.lucene.index.TestIndexWriterOnVMError$3.eval(TestIndexWriterOnVMError.java:270)
   [junit4]   1> 	at org.apa

[...truncated too long message...]

EST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[8C1E58940135899E]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 9158573 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[8C1E58940135899E]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@3284d0f1: rootName = null, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@1d5100
   [junit4]   2> 9158576 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[8C1E58940135899E]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 9158577 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[8C1E58940135899E]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@4672aa43: rootName = null, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@5cc3c634
   [junit4]   2> 9158577 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[8C1E58940135899E]) [    ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:39036_
   [junit4]   2> 9158578 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[8C1E58940135899E]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 9158579 INFO  (zkCallback-10620-thread-1-processing-n:127.0.0.1:39036_) [n:127.0.0.1:39036_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
   [junit4]   2> 9158579 INFO  (zkCallback-10625-thread-1-processing-n:127.0.0.1:39036_) [n:127.0.0.1:39036_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
   [junit4]   2> 9158580 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[8C1E58940135899E]) [    ] o.a.s.c.Overseer Overseer (id=72315250603458599-127.0.0.1:39036_-n_0000000007) closing
   [junit4]   2> 9158580 INFO  (OverseerStateUpdate-72315250603458599-127.0.0.1:39036_-n_0000000007) [n:127.0.0.1:39036_    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:39036_
   [junit4]   2> 9158580 WARN  (OverseerAutoScalingTriggerThread-72315250603458599-127.0.0.1:39036_-n_0000000007) [n:127.0.0.1:39036_    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 9158583 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[8C1E58940135899E]) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@306c6875{SSL,[ssl, http/1.1]}{127.0.0.1:0}
   [junit4]   2> 9158584 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[8C1E58940135899E]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@52092334{/,null,UNAVAILABLE}
   [junit4]   2> 9158584 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[8C1E58940135899E]) [    ] o.e.j.s.session Stopped scavenging
   [junit4]   2> 9158584 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[8C1E58940135899E]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testBatchAddsWithDelete
   [junit4]   2> 9158584 ERROR (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[8C1E58940135899E]) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 9158585 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[8C1E58940135899E]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:37459 37459
   [junit4]   2> 9158654 INFO  (Thread-64967) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:37459 37459
   [junit4]   2> 9158655 WARN  (Thread-64967) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	53	/solr/collections/tmp_collection/terms/shard2
   [junit4]   2> 	52	/solr/collections/tmp_collection/terms/shard1
   [junit4]   2> 	38	/solr/aliases.json
   [junit4]   2> 	14	/solr/configs/conf1
   [junit4]   2> 	8	/solr/security.json
   [junit4]   2> 	8	/solr/collections/target_collection/terms/shard1
   [junit4]   2> 	7	/solr/collections/source_collection/terms/shard1
   [junit4]   2> 	7	/solr/collections/source_collection/terms/shard2
   [junit4]   2> 	7	/solr/collections/target_collection/terms/shard2
   [junit4]   2> 	4	/solr/collections/tmp_collection/leaders/shard2/leader
   [junit4]   2> 	4	/solr/collections/tmp_collection/leaders/shard1/leader
   [junit4]   2> 	2	/solr/collections/source_collection/leaders/shard2/leader
   [junit4]   2> 	2	/solr/collections/source_collection/leaders/shard1/leader
   [junit4]   2> 	2	/solr/collections/target_collection/leaders/shard2/leader
   [junit4]   2> 	2	/solr/collections/target_collection/leaders/shard1/leader
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	38	/solr/clusterprops.json
   [junit4]   2> 	38	/solr/clusterstate.json
   [junit4]   2> 	12	/solr/collections/tmp_collection/cdcr/state/process
   [junit4]   2> 	12	/solr/collections/tmp_collection/state.json
   [junit4]   2> 	12	/solr/collections/tmp_collection/cdcr/state/buffer
   [junit4]   2> 	7	/solr/autoscaling.json
   [junit4]   2> 	5	/solr/collections/source_collection/cdcr/state/process
   [junit4]   2> 	4	/solr/collections/target_collection/cdcr/state/buffer
   [junit4]   2> 	4	/solr/collections/source_collection/state.json
   [junit4]   2> 	4	/solr/collections/source_collection/cdcr/state/buffer
   [junit4]   2> 	4	/solr/collections/target_collection/cdcr/state/process
   [junit4]   2> 	4	/solr/collections/target_collection/state.json
   [junit4]   2> 	3	/solr/collections/tmp_collection/leader_elect/shard1/election/72315250603458565-core_node42-n_0000000000
   [junit4]   2> 	2	/solr/collections/tmp_collection/leader_elect/shard2/election/72315250603458569-core_node44-n_0000000000
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	38	/solr/live_nodes
   [junit4]   2> 	37	/solr/collections
   [junit4]   2> 	7	/solr/overseer/queue
   [junit4]   2> 	7	/solr/autoscaling/events/.scheduled_maintenance
   [junit4]   2> 	7	/solr/overseer/collection-queue-work
   [junit4]   2> 	7	/solr/overseer/queue-work
   [junit4]   2> 	6	/solr/autoscaling/events/.auto_add_replicas
   [junit4]   2> 
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrReplicationDistributedZkTest_8C1E58940135899E-001
   [junit4]   2> Mar 21, 2018 1:03:15 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 5 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarity(queryNorm=true): {}, locale=lt, timezone=Pacific/Pago_Pago
   [junit4]   2> NOTE: Linux 4.4.0-112-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=1,free=66199328,total=521666560
   [junit4]   2> NOTE: All tests run in this JVM: [TestCloudPivotFacet, MultiTermTest, MinimalSchemaTest, TestBadConfig, HdfsCollectionsAPIDistributedZkTest, DeleteShardTest, TestRetrieveFieldsOptimizer, UniqFieldsUpdateProcessorFactoryTest, URLClassifyProcessorTest, DirectSolrSpellCheckerTest, TestLegacyFieldReuse, TestHalfAndHalfDocValues, ZkFailoverTest, TestIntervalFaceting, DeleteLastCustomShardedReplicaTest, TestAddFieldRealTimeGet, HdfsBasicDistributedZkTest, TimeZoneUtilsTest, TestSolrCloudWithKerberosAlt, ConvertedLegacyTest, DirectoryFactoryTest, ComputePlanActionTest, PingRequestHandlerTest, TestDownShardTolerantSearch, TestDistribIDF, HdfsTlogReplayBufferedWhileIndexingTest, TestManagedResource, SolrTestCaseJ4Test, TestDFRSimilarityFactory, MetricsConfigTest, TestUtils, DistributedFacetPivotLongTailTest, SolrMetricReporterTest, TestSolrConfigHandlerCloud, HdfsRestartWhileUpdatingTest, TestRealTimeGet, TestMultiValuedNumericRangeQuery, DistributedSuggestComponentTest, TestDistribStateManager, RulesTest, TestMiniSolrCloudClusterSSL, TestApiFramework, TestAnalyzedSuggestions, TestQueryWrapperFilter, NotRequiredUniqueKeyTest, BooleanFieldTest, TestJettySolrRunner, PKIAuthenticationIntegrationTest, TestReRankQParserPlugin, DocValuesMultiTest, UUIDUpdateProcessorFallbackTest, SharedFSAutoReplicaFailoverTest, DistanceFunctionTest, StatelessScriptUpdateProcessorFactoryTest, V2ApiIntegrationTest, UpdateLogTest, RegexBytesRefFilterTest, TestRandomRequestDistribution, TestCoreAdminApis, SuggestComponentContextFilterQueryTest, TestEmbeddedSolrServerAdminHandler, TestSolrDeletionPolicy2, TestLFUCache, TestPullReplica, CdcrVersionReplicationTest, TestClassicSimilarityFactory, CircularListTest, OverseerStatusTest, TestNamedUpdateProcessors, PreAnalyzedUpdateProcessorTest, TestMacros, CopyFieldTest, SearchRateTriggerTest, UnloadDistributedZkTest, CdcrUpdateLogTest, BlockJoinFacetSimpleTest, TestAtomicUpdateErrorCases, RollingRestartTest, ClusterStateTest, TestSuggestSpellingConverter, TestLeaderElectionZkExpiry, TestStressInPlaceUpdates, AutoAddReplicasIntegrationTest, HdfsNNFailoverTest, CacheHeaderTest, TestUninvertingReader, TestRestoreCore, TestSolrXml, TestFieldTypeCollectionResource, TestCoreDiscovery, TestSSLRandomization, SuggesterTSTTest, TestSerializedLuceneMatchVersion, ReplaceNodeTest, SolrSlf4jReporterTest, TestCloudDeleteByQuery, HdfsDirectoryTest, DisMaxRequestHandlerTest, HttpSolrCallGetCoreTest, CdcrReplicationDistributedZkTest]
   [junit4] Completed [613/784 (8!)] on J2 in 1836.16s, 11 tests, 1 failure <<< FAILURES!

[...truncated 50059 lines...]

[JENKINS] Lucene-Solr-BadApples-NightlyTests-7.x - Build # 2 - Still Unstable

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-BadApples-NightlyTests-7.x/2/

10 tests failed.
FAILED:  org.apache.solr.cloud.RestartWhileUpdatingTest.test

Error Message:
There are still nodes recoverying - waited for 320 seconds

Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 320 seconds
	at __randomizedtesting.SeedInfo.seed([AF42AF945BE7BEEA:2716904EF51BD312]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:185)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:921)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1478)
	at org.apache.solr.cloud.RestartWhileUpdatingTest.test(RestartWhileUpdatingTest.java:144)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates.test_dv_idx

Error Message:
Some docs had errors -- check logs expected:<0> but was:<3>

Stack Trace:
java.lang.AssertionError: Some docs had errors -- check logs expected:<0> but was:<3>
	at __randomizedtesting.SeedInfo.seed([AF42AF945BE7BEEA:3A3EDA5C873B130B]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates.checkField(TestStressCloudBlindAtomicUpdates.java:342)
	at org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates.test_dv_idx(TestStressCloudBlindAtomicUpdates.java:231)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates.test_dv_stored_idx

Error Message:
expected:<453> but was:<373>

Stack Trace:
java.lang.AssertionError: expected:<453> but was:<373>
	at __randomizedtesting.SeedInfo.seed([AF42AF945BE7BEEA:A565FEB424911DBD]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates.checkField(TestStressCloudBlindAtomicUpdates.java:272)
	at org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates.test_dv_stored_idx(TestStressCloudBlindAtomicUpdates.java:221)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates.test_dv

Error Message:
expected:<534> but was:<222>

Stack Trace:
java.lang.AssertionError: expected:<534> but was:<222>
	at __randomizedtesting.SeedInfo.seed([AF42AF945BE7BEEA:9956CDD2D1BA84FB]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates.checkField(TestStressCloudBlindAtomicUpdates.java:272)
	at org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates.test_dv(TestStressCloudBlindAtomicUpdates.java:198)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates.test_dv_stored

Error Message:
expected:<486> but was:<303>

Stack Trace:
java.lang.AssertionError: expected:<486> but was:<303>
	at __randomizedtesting.SeedInfo.seed([AF42AF945BE7BEEA:3D7ECE10E7A08332]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates.checkField(TestStressCloudBlindAtomicUpdates.java:272)
	at org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates.test_dv_stored(TestStressCloudBlindAtomicUpdates.java:210)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.TestTlogReplica.testRecovery

Error Message:
Can not find doc 8 in https://127.0.0.1:39913/solr

Stack Trace:
java.lang.AssertionError: Can not find doc 8 in https://127.0.0.1:39913/solr
	at __randomizedtesting.SeedInfo.seed([AF42AF945BE7BEEA:6EB2D63876B7744D]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNotNull(Assert.java:526)
	at org.apache.solr.cloud.TestTlogReplica.checkRTG(TestTlogReplica.java:889)
	at org.apache.solr.cloud.TestTlogReplica.testRecovery(TestTlogReplica.java:603)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.update.processor.AtomicUpdateProcessorFactoryTest.testMultipleThreads

Error Message:
Exception during query

Stack Trace:
java.lang.RuntimeException: Exception during query
	at __randomizedtesting.SeedInfo.seed([AF42AF945BE7BEEA:8370D819D723C86E]:0)
	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:908)
	at org.apache.solr.update.processor.AtomicUpdateProcessorFactoryTest.testMultipleThreads(AtomicUpdateProcessorFactoryTest.java:260)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.RuntimeException: REQUEST FAILED: xpath=//result[@numFound=1]
	xml response was: <?xml version="1.0" encoding="UTF-8"?>
<response>
<lst name="responseHeader"><int name="status">0</int><int name="QTime">0</int></lst><result name="response" numFound="0" start="0"></result>
</response>

	request was:q=int_i:19&wt=xml
	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:901)
	... 40 more


FAILED:  org.apache.solr.ltr.TestLTRReRankingPipeline.testDifferentTopN

Error Message:
expected:<1.0> but was:<0.0>

Stack Trace:
java.lang.AssertionError: expected:<1.0> but was:<0.0>
	at __randomizedtesting.SeedInfo.seed([44690DD45D5398:F1E51B5DE1E6990A]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:443)
	at org.junit.Assert.assertEquals(Assert.java:512)
	at org.apache.solr.ltr.TestLTRReRankingPipeline.testDifferentTopN(TestLTRReRankingPipeline.java:255)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.hdfs.HdfsUnloadDistributedZkTest.test

Error Message:
Captured an uncaught exception in thread: Thread[id=12930, name=testExecutor-4156-thread-15, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=12930, name=testExecutor-4156-thread-15, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest]
Caused by: java.lang.RuntimeException: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:37599
	at __randomizedtesting.SeedInfo.seed([AF42AF945BE7BEEA]:0)
	at org.apache.solr.cloud.BasicDistributedZkTest.lambda$createCollectionInOneInstance$0(BasicDistributedZkTest.java:596)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:37599
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:654)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
	at org.apache.solr.cloud.BasicDistributedZkTest.lambda$createCollectionInOneInstance$0(BasicDistributedZkTest.java:594)
	... 4 more
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:171)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:542)
	... 9 more


FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestIndexFetchOnMasterRestart

Error Message:
IOException occured when talking to server at: http://127.0.0.1:37348/solr/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:37348/solr/collection1
	at __randomizedtesting.SeedInfo.seed([AF42AF945BE7BEEA:77B56B70F03C7CB6]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:657)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:173)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:138)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:152)
	at org.apache.solr.handler.TestReplicationHandler.index(TestReplicationHandler.java:180)
	at org.apache.solr.handler.TestReplicationHandler.doTestIndexFetchOnMasterRestart(TestReplicationHandler.java:643)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:37348 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:141)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:542)
	... 47 more




Build Log:
[...truncated 2012 lines...]
   [junit4] JVM J0: stdout was not empty, see: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/lucene/build/core/test/temp/junit4-J0-20180313_191151_4125963434727438239240.sysout
   [junit4] >>> JVM J0 emitted unexpected output (verbatim) ----
   [junit4] codec: Asserting, pf: BloomFilter, dvf: Lucene70
   [junit4] <<< JVM J0: EOF ----

[...truncated 10883 lines...]
   [junit4] Suite: org.apache.solr.update.processor.AtomicUpdateProcessorFactoryTest
   [junit4]   2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.update.processor.AtomicUpdateProcessorFactoryTest_AF42AF945BE7BEEA-001/init-core-data-001
   [junit4]   2> 22346 WARN  (SUITE-AtomicUpdateProcessorFactoryTest-seed#[AF42AF945BE7BEEA]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=3 numCloses=3
   [junit4]   2> 22346 INFO  (SUITE-AtomicUpdateProcessorFactoryTest-seed#[AF42AF945BE7BEEA]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 22361 INFO  (SUITE-AtomicUpdateProcessorFactoryTest-seed#[AF42AF945BE7BEEA]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 22362 INFO  (SUITE-AtomicUpdateProcessorFactoryTest-seed#[AF42AF945BE7BEEA]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 22363 INFO  (SUITE-AtomicUpdateProcessorFactoryTest-seed#[AF42AF945BE7BEEA]-worker) [    ] o.a.s.SolrTestCaseJ4 ####initCore
   [junit4]   2> 22363 INFO  (SUITE-AtomicUpdateProcessorFactoryTest-seed#[AF42AF945BE7BEEA]-worker) [    ] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/lib, /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 23069 INFO  (SUITE-AtomicUpdateProcessorFactoryTest-seed#[AF42AF945BE7BEEA]-worker) [    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.4.0
   [junit4]   2> 23729 INFO  (SUITE-AtomicUpdateProcessorFactoryTest-seed#[AF42AF945BE7BEEA]-worker) [    ] o.a.s.s.IndexSchema [null] Schema name=test
   [junit4]   2> 24215 WARN  (SUITE-AtomicUpdateProcessorFactoryTest-seed#[AF42AF945BE7BEEA]-worker) [    ] o.a.s.c.SolrResourceLoader Solr loaded a deprecated plugin/analysis class [solr.WordDelimiterFilterFactory]. Please consult documentation how to replace it accordingly.
   [junit4]   2> 24306 WARN  (SUITE-AtomicUpdateProcessorFactoryTest-seed#[AF42AF945BE7BEEA]-worker) [    ] o.a.s.c.SolrResourceLoader Solr loaded a deprecated plugin/analysis class [solr.GeoHashField]. Please consult documentation how to replace it accordingly.
   [junit4]   2> 24313 WARN  (SUITE-AtomicUpdateProcessorFactoryTest-seed#[AF42AF945BE7BEEA]-worker) [    ] o.a.s.c.SolrResourceLoader Solr loaded a deprecated plugin/analysis class [solr.LatLonType]. Please consult documentation how to replace it accordingly.
   [junit4]   2> 24321 WARN  (SUITE-AtomicUpdateProcessorFactoryTest-seed#[AF42AF945BE7BEEA]-worker) [    ] o.a.s.c.SolrResourceLoader Solr loaded a deprecated plugin/analysis class [solr.CurrencyField]. Please consult documentation how to replace it accordingly.
   [junit4]   2> 24425 INFO  (SUITE-AtomicUpdateProcessorFactoryTest-seed#[AF42AF945BE7BEEA]-worker) [    ] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 24616 WARN  (SUITE-AtomicUpdateProcessorFactoryTest-seed#[AF42AF945BE7BEEA]-worker) [    ] o.a.s.m.r.SolrJmxReporter No JMX server found. Not exposing Solr metrics via JMX.
   [junit4]   2> 24616 WARN  (SUITE-AtomicUpdateProcessorFactoryTest-seed#[AF42AF945BE7BEEA]-worker) [    ] o.a.s.m.r.SolrJmxReporter No JMX server found. Not exposing Solr metrics via JMX.
   [junit4]   2> 24616 WARN  (SUITE-AtomicUpdateProcessorFactoryTest-seed#[AF42AF945BE7BEEA]-worker) [    ] o.a.s.m.r.SolrJmxReporter No JMX server found. Not exposing Solr metrics via JMX.
   [junit4]   2> 24644 INFO  (coreLoadExecutor-18-thread-1) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 24645 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/lib, /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 24706 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.4.0
   [junit4]   2> 24780 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 24938 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 24981 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1, trusted=true
   [junit4]   2> 24981 WARN  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter No JMX server found. Not exposing Solr metrics via JMX.
   [junit4]   2> 24994 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 24994 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.update.processor.AtomicUpdateProcessorFactoryTest_AF42AF945BE7BEEA-001/init-core-data-001/]
   [junit4]   2> 25009 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=13, maxMergeAtOnceExplicit=31, maxMergedSegmentMB=69.2470703125, floorSegmentMB=1.44921875, forceMergeDeletesPctAllowed=28.317515416449137, segmentsPerTier=39.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7542165326937235
   [junit4]   2> 25030 WARN  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.c.SolrResourceLoader Solr loaded a deprecated plugin/analysis class [solr.OldLuceneQParserPlugin]. Please consult documentation how to replace it accordingly.
   [junit4]   2> 25298 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 25298 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 25319 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 25320 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 25321 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=22, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1692940809660417]
   [junit4]   2> 25330 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@298565e[collection1] main]
   [junit4]   2> 25333 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
   [junit4]   2> 25405 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.s.DirectSolrSpellChecker init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 25542 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 25542 INFO  (searcherExecutor-19-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
   [junit4]   2> 25550 INFO  (searcherExecutor-19-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: direct
   [junit4]   2> 25550 INFO  (searcherExecutor-19-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
   [junit4]   2> 25550 INFO  (searcherExecutor-19-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: multipleFields
   [junit4]   2> 25551 INFO  (searcherExecutor-19-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 25554 INFO  (searcherExecutor-19-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: external
   [junit4]   2> 25555 INFO  (searcherExecutor-19-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: freq
   [junit4]   2> 25555 INFO  (searcherExecutor-19-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: fqcn
   [junit4]   2> 25556 INFO  (searcherExecutor-19-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: perDict
   [junit4]   2> 25557 INFO  (searcherExecutor-19-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@298565e[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 25558 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1594874355499663360
   [junit4]   2> 25565 INFO  (SUITE-AtomicUpdateProcessorFactoryTest-seed#[AF42AF945BE7BEEA]-worker) [    ] o.a.s.SolrTestCaseJ4 ####initCore end
   [junit4]   2> 25567 INFO  (TEST-AtomicUpdateProcessorFactoryTest.testBasics-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testBasics
   [junit4]   2> 25627 INFO  (TEST-AtomicUpdateProcessorFactoryTest.testBasics-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1594874355572015104,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 25627 INFO  (TEST-AtomicUpdateProcessorFactoryTest.testBasics-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@74f44ea5 commitCommandVersion:1594874355572015104
   [junit4]   2> 25726 INFO  (TEST-AtomicUpdateProcessorFactoryTest.testBasics-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@75a403ac[collection1] main]
   [junit4]   2> 25727 INFO  (TEST-AtomicUpdateProcessorFactoryTest.testBasics-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 25727 INFO  (searcherExecutor-19-thread-1) [    ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@75a403ac[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.4.0):C1)))}
   [junit4]   2> 25728 INFO  (TEST-AtomicUpdateProcessorFactoryTest.testBasics-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{commit=} 0 100
   [junit4]   2> 25743 INFO  (TEST-AtomicUpdateProcessorFactoryTest.testBasics-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:1&wt=xml} hits=1 status=0 QTime=9
   [junit4]   2> 25756 INFO  (TEST-AtomicUpdateProcessorFactoryTest.testBasics-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=cat:human&wt=xml} hits=1 status=0 QTime=0
   [junit4]   2> 25758 INFO  (TEST-AtomicUpdateProcessorFactoryTest.testBasics-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=title:Mr&wt=xml} hits=1 status=0 QTime=0
   [junit4]   2> 25767 INFO  (TEST-AtomicUpdateProcessorFactoryTest.testBasics-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=count_i:20&wt=xml} hits=1 status=0 QTime=8
   [junit4]   2> 25769 INFO  (TEST-AtomicUpdateProcessorFactoryTest.testBasics-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=name_s:Virat&wt=xml} hits=1 status=0 QTime=0
   [junit4]   2> 25772 INFO  (TEST-AtomicUpdateProcessorFactoryTest.testBasics-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=multiDefault:Delhi&wt=xml} hits=1 status=0 QTime=0
   [junit4]   2> 25790 INFO  (TEST-AtomicUpdateProcessorFactoryTest.testBasics-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1594874355742932992,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 25791 INFO  (TEST-AtomicUpdateProcessorFactoryTest.testBasics-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@74f44ea5 commitCommandVersion:1594874355742932992
   [junit4]   2> 25817 INFO  (TEST-AtomicUpdateProcessorFactoryTest.testBasics-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@ca206d9[collection1] main]
   [junit4]   2> 25819 INFO  (searcherExecutor-19-thread-1) [    ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@ca206d9[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_1(7.4.0):C1)))}
   [junit4]   2> 25827 INFO  (TEST-AtomicUpdateProcessorFactoryTest.testBasics-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 25829 INFO  (TEST-AtomicUpdateProcessorFactoryTest.testBasics-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{commit=} 0 38
   [junit4]   2> 25834 INFO  (TEST-AtomicUpdateProcessorFactoryTest.testBasics-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:1&wt=xml} hits=1 status=0 QTime=4
   [junit4]   2> 25839 INFO  (TEST-AtomicUpdateProcessorFactoryTest.testBasics-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=cat:human&wt=xml} hits=1 status=0 QTime=0
   [junit4]   2> 25841 INFO  (TEST-AtomicUpdateProcessorFactoryTest.testBasics-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=cat:animal&wt=xml} hits=1 status=0 QTime=0
   [junit4]   2> 25844 INFO  (TEST-AtomicUpdateProcessorFactoryTest.testBasics-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=title:Mr&wt=xml} hits=0 status=0 QTime=0
   [junit4]   2> 25845 INFO  (TEST-AtomicUpdateProcessorFactoryTest.testBasics-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=title:Dr&wt=xml} hits=1 status=0 QTime=0
   [junit4]   2> 25945 INFO  (TEST-AtomicUpdateProcessorFactoryTest.testBasics-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=count_i:20&wt=xml} hits=0 status=0 QTime=99
   [junit4]   2> 25947 INFO  (TEST-AtomicUpdateProcessorFactoryTest.testBasics-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=count_i:40&wt=xml} hits=1 status=0 QTime=0
   [junit4]   2> 25948 INFO  (TEST-AtomicUpdateProcessorFactoryTest.testBasics-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=name_s:Virat&wt=xml} hits=0 status=0 QTime=0
   [junit4]   2> 25949 INFO  (TEST-AtomicUpdateProcessorFactoryTest.testBasics-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=multiDefault:Delhi&wt=xml} hits=0 status=0 QTime=0
   [junit4]   2> 25958 INFO  (TEST-AtomicUpdateProcessorFactoryTest.testBasics-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testBasics
   [junit4]   2> 25964 INFO  (TEST-AtomicUpdateProcessorFactoryTest.testMultipleThreads-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testMultipleThreads
   [junit4]   2> 25970 INFO  (TEST-AtomicUpdateProcessorFactoryTest.testMultipleThreads-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.u.DirectUpdateHandler2 [collection1] REMOVING ALL DOCUMENTS FROM INDEX
   [junit4]   2> 25973 INFO  (TEST-AtomicUpdateProcessorFactoryTest.testMultipleThreads-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@3e4ed172[collection1] realtime]
   [junit4]   2> 25974 INFO  (TEST-AtomicUpdateProcessorFactoryTest.testMultipleThreads-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={_version_=-9223372036854775807&update.distrib=FROMLEADER&versions=true&wt=json&indent=true}{deleteByQuery=*:* (-9223372036854775807)} 0 9
   [junit4]   2> 25988 WARN  (Thread-7) [    ] o.a.s.u.p.AtomicUpdateProcessorFactory Atomic update failed due to version conflict for 10 expected=1594874355941113856 actual=1594874355950551040Retrying with new version .... (1)
   [junit4]   2> 25989 WARN  (Thread-12) [    ] o.a.s.u.p.AtomicUpdateProcessorFactory Atomic update failed due to version conflict for 10 expected=1594874355941113856 actual=1594874355950551040Retrying with new version .... (1)
   [junit4]   2> 25996 WARN  (Thread-10) [    ] o.a.s.u.p.AtomicUpdateProcessorFactory Atomic update failed due to version conflict for 10 expected=1594874355941113856 actual=1594874355957891072Retrying with new version .... (1)
   [junit4]   2> 25997 WARN  (Thread-11) [    ] o.a.s.u.p.AtomicUpdateProcessorFactory Atomic update failed due to version conflict for 10 expected=1594874355941113856 actual=1594874355959988224Retrying with new version .... (1)
   [junit4]   2> 25998 WARN  (Thread-9) [    ] o.a.s.u.p.AtomicUpdateProcessorFactory Atomic update failed due to version conflict for 10 expected=1594874355941113856 actual=1594874355959988225Retrying with new version .... (1)
   [junit4]   2> 25998 WARN  (Thread-8) [    ] o.a.s.u.p.AtomicUpdateProcessorFactory Atomic update failed due to version conflict for 10 expected=1594874355941113856 actual=1594874355959988225Retrying with new version .... (1)
   [junit4]   2> 25999 WARN  (Thread-6) [    ] o.a.s.u.p.AtomicUpdateProcessorFactory Atomic update failed due to version conflict for 10 expected=1594874355941113856 actual=1594874355959988225Retrying with new version .... (1)
   [junit4]   2> 25999 WARN  (Thread-9) [    ] o.a.s.u.p.AtomicUpdateProcessorFactory Atomic update failed due to version conflict for 10 expected=1594874355959988225 actual=1594874355962085376Retrying with new version .... (2)
   [junit4]   2> 26000 WARN  (Thread-6) [    ] o.a.s.u.p.AtomicUpdateProcessorFactory Atomic update failed due to version conflict for 10 expected=1594874355959988225 actual=1594874355962085376Retrying with new version .... (2)
   [junit4]   2> 26006 INFO  (TEST-AtomicUpdateProcessorFactoryTest.testMultipleThreads-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1594874355969425408,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 26006 INFO  (TEST-AtomicUpdateProcessorFactoryTest.testMultipleThreads-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@74f44ea5 commitCommandVersion:1594874355969425408
   [junit4]   2> 26018 INFO  (TEST-AtomicUpdateProcessorFactoryTest.testMultipleThreads-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@318b00f6[collection1] main]
   [junit4]   2> 26019 INFO  (TEST-AtomicUpdateProcessorFactoryTest.testMultipleThreads-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 26020 INFO  (searcherExecutor-19-thread-1) [    ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@318b00f6[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_2(7.4.0):C10/9:delGen=1)))}
   [junit4]   2> 26021 INFO  (TEST-AtomicUpdateProcessorFactoryTest.testMultipleThreads-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{commit=} 0 15
   [junit4]   2> 26023 INFO  (TEST-AtomicUpdateProcessorFactoryTest.testMultipleThreads-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:10&wt=xml} hits=1 status=0 QTime=1
   [junit4]   2> 26052 INFO  (TEST-AtomicUpdateProcessorFactoryTest.testMultipleThreads-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=cat:dhmz7ae33h7drvxvu72r+nas1rtsqhf9c6gnlbc8t+9kg385vw71vxn6ebutuq+h05hvlo9ld4samo4ru6n+ldma40w7libxbs6irrlm&wt=xml} hits=1 status=0 QTime=11
   [junit4]   2> 26053 INFO  (TEST-AtomicUpdateProcessorFactoryTest.testMultipleThreads-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=int_i:19&wt=xml} hits=0 status=0 QTime=0
   [junit4]   2> 26058 ERROR (TEST-AtomicUpdateProcessorFactoryTest.testMultipleThreads-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.SolrTestCaseJ4 REQUEST FAILED: xpath=//result[@numFound=1]
   [junit4]   2> 	xml response was: <?xml version="1.0" encoding="UTF-8"?>
   [junit4]   2> <response>
   [junit4]   2> <lst name="responseHeader"><int name="status">0</int><int name="QTime">0</int></lst><result name="response" numFound="0" start="0"></result>
   [junit4]   2> </response>
   [junit4]   2> 
   [junit4]   2> 	request was:q=int_i:19&wt=xml
   [junit4]   2> 26059 ERROR (TEST-AtomicUpdateProcessorFactoryTest.testMultipleThreads-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.SolrTestCaseJ4 REQUEST FAILED: q=int_i:19&wt=xml:java.lang.RuntimeException: REQUEST FAILED: xpath=//result[@numFound=1]
   [junit4]   2> 	xml response was: <?xml version="1.0" encoding="UTF-8"?>
   [junit4]   2> <response>
   [junit4]   2> <lst name="responseHeader"><int name="status">0</int><int name="QTime">0</int></lst><result name="response" numFound="0" start="0"></result>
   [junit4]   2> </response>
   [junit4]   2> 
   [junit4]   2> 	request was:q=int_i:19&wt=xml
   [junit4]   2> 	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:901)
   [junit4]   2> 	at org.apache.solr.update.processor.AtomicUpdateProcessorFactoryTest.testMultipleThreads(AtomicUpdateProcessorFactoryTest.java:260)
   [junit4]   2> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2> 	at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
   [junit4]   2> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 
   [junit4]   2> 26066 INFO  (TEST-AtomicUpdateProcessorFactoryTest.testMultipleThreads-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testMultipleThreads
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=AtomicUpdateProcessorFactoryTest -Dtests.method=testMultipleThreads -Dtests.seed=AF42AF945BE7BEEA -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.badapples=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=he-IL -Dtests.timezone=Pacific/Guam -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   0.11s J1 | AtomicUpdateProcessorFactoryTest.testMultipleThreads <<<
   [junit4]    > Throwable #1: java.lang.RuntimeException: Exception during query
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([AF42AF945BE7BEEA:8370D819D723C86E]:0)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:908)
   [junit4]    > 	at org.apache.solr.update.processor.AtomicUpdateProcessorFactoryTest.testMultipleThreads(AtomicUpdateProcessorFactoryTest.java:260)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > Caused by: java.lang.RuntimeException: REQUEST FAILED: xpath=//result[@numFound=1]
   [junit4]    > 	xml response was: <?xml version="1.0" encoding="UTF-8"?>
   [junit4]    > <response>
   [junit4]    > <lst name="responseHeader"><int name="status">0</int><int name="QTime">0</int></lst><result name="response" numFound="0" start="0"></result>
   [junit4]    > </response>
   [junit4]    > 	request was:q=int_i:19&wt=xml
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:901)
   [junit4]    > 	... 40 more
   [junit4]   2> 26073 INFO  (TEST-AtomicUpdateProcessorFactoryTest.testWrongAtomicOpPassed-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testWrongAtomicOpPassed
   [junit4]   2> 26074 INFO  (TEST-AtomicUpdateProcessorFactoryTest.testWrongAtomicOpPassed-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testWrongAtomicOpPassed
   [junit4]   2> 26085 INFO  (TEST-AtomicUpdateProcessorFactoryTest.testNoUniqueIdPassed-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testNoUniqueIdPassed
   [junit4]   2> 26085 INFO  (TEST-AtomicUpdateProcessorFactoryTest.testNoUniqueIdPassed-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testNoUniqueIdPassed
   [junit4]   2> 26085 INFO  (SUITE-AtomicUpdateProcessorFactoryTest-seed#[AF42AF945BE7BEEA]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> 26086 INFO  (SUITE-AtomicUpdateProcessorFactoryTest-seed#[AF42AF945BE7BEEA]-worker) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1092952624
   [junit4]   2> 26086 INFO  (SUITE-AtomicUpdateProcessorFactoryTest-seed#[AF42AF945BE7BEEA]-worker) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 26086 INFO  (SUITE-AtomicUpdateProcessorFactoryTest-seed#[AF42AF945BE7BEEA]-worker) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@747db93a: rootName = null, domain = solr.node, service url = null, agent id = null] for registry solr.node / null
   [junit4]   2> 26086 INFO  (SUITE-AtomicUpdateProcessorFactoryTest-seed#[AF42AF945BE7BEEA]-worker) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 26086 INFO  (SUITE-AtomicUpdateProcessorFactoryTest-seed#[AF42AF945BE7BEEA]-worker) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@6ac24efa: rootName = null, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / null
   [junit4]   2> 26086 INFO  (SUITE-AtomicUpdateProcessorFactoryTest-seed#[AF42AF945BE7BEEA]-worker) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 26086 INFO  (SUITE-AtomicUpdateProcessorFactoryTest-seed#[AF42AF945BE7BEEA]-worker) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@3128e746: rootName = null, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / null
   [junit4]   2> 26092 INFO  (coreCloseExecutor-24-thread-1) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@543364e6
   [junit4]   2> 26092 INFO  (coreCloseExecutor-24-thread-1) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=1412654310
   [junit4]   2> 26092 INFO  (coreCloseExecutor-24-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@73cdaeba: rootName = null, domain = solr.core.collection1, service url = null, agent id = null] for registry solr.core.collection1 / null
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.update.processor.AtomicUpdateProcessorFactoryTest_AF42AF945BE7BEEA-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70), sim=RandomSimilarity(queryNorm=true): {}, locale=he-IL, timezone=Pacific/Guam
   [junit4]   2> NOTE: Linux 4.4.0-112-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=1,free=296661440,total=341311488
   [junit4]   2> NOTE: All tests run in this JVM: [DirectSolrConnectionTest, AtomicUpdateProcessorFactoryTest]
   [junit4] Completed [3/783 (1!)] on J1 in 3.87s, 4 tests, 1 error <<< FAILURES!

[...truncated 856 lines...]
   [junit4] Suite: org.apache.solr.cloud.hdfs.HdfsUnloadDistributedZkTest
   [junit4]   2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_AF42AF945BE7BEEA-001/init-core-data-001
   [junit4]   2> 2735397 WARN  (SUITE-HdfsUnloadDistributedZkTest-seed#[AF42AF945BE7BEEA]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=2 numCloses=2
   [junit4]   2> 2735399 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[AF42AF945BE7BEEA]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 2735400 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[AF42AF945BE7BEEA]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
   [junit4]   2> 2735400 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[AF42AF945BE7BEEA]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 2735400 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[AF42AF945BE7BEEA]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
   [junit4]   1> Formatting using clusterid: testClusterID
   [junit4]   2> 2735497 WARN  (SUITE-HdfsUnloadDistributedZkTest-seed#[AF42AF945BE7BEEA]-worker) [    ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
   [junit4]   2> 2735527 WARN  (SUITE-HdfsUnloadDistributedZkTest-seed#[AF42AF945BE7BEEA]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 2735533 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[AF42AF945BE7BEEA]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 2735549 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[AF42AF945BE7BEEA]-worker) [    ] o.m.log Extract jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/hdfs to ./temp/Jetty_lucene2.us.west_apache_org_33930_hdfs____xnqo2o/webapp
   [junit4]   2> 2736371 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[AF42AF945BE7BEEA]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@lucene2-us-west.apache.org:33930
   [junit4]   2> 2736687 WARN  (SUITE-HdfsUnloadDistributedZkTest-seed#[AF42AF945BE7BEEA]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 2736688 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[AF42AF945BE7BEEA]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 2736701 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[AF42AF945BE7BEEA]-worker) [    ] o.m.log Extract jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_42948_datanode____6bfr1i/webapp
   [junit4]   2> 2737233 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[AF42AF945BE7BEEA]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:42948
   [junit4]   2> 2737563 WARN  (SUITE-HdfsUnloadDistributedZkTest-seed#[AF42AF945BE7BEEA]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 2737564 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[AF42AF945BE7BEEA]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 2737606 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[AF42AF945BE7BEEA]-worker) [    ] o.m.log Extract jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_37834_datanode____fl7h24/webapp
   [junit4]   2> 2737878 ERROR (DataNode: [[[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_AF42AF945BE7BEEA-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_AF42AF945BE7BEEA-001/tempDir-001/hdfsBaseDir/data/data2/]]  heartbeating to lucene2-us-west.apache.org/127.0.0.1:37818) [    ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
   [junit4]   2> 2737917 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0xbd55df04fd7bc: from storage DS-f7683840-51cf-4e05-84fa-62e8a11c92fd node DatanodeRegistration(127.0.0.1:35189, datanodeUuid=6ff41209-beee-4f57-848b-6f1892cd7b98, infoPort=36891, infoSecurePort=0, ipcPort=44292, storageInfo=lv=-56;cid=testClusterID;nsid=686266960;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
   [junit4]   2> 2737917 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0xbd55df04fd7bc: from storage DS-0917029f-c0b2-4a34-8620-3613e8d84a56 node DatanodeRegistration(127.0.0.1:35189, datanodeUuid=6ff41209-beee-4f57-848b-6f1892cd7b98, infoPort=36891, infoSecurePort=0, ipcPort=44292, storageInfo=lv=-56;cid=testClusterID;nsid=686266960;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 2738201 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[AF42AF945BE7BEEA]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:37834
   [junit4]   2> 2738618 ERROR (DataNode: [[[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_AF42AF945BE7BEEA-001/tempDir-001/hdfsBaseDir/data/data3/, [DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_AF42AF945BE7BEEA-001/tempDir-001/hdfsBaseDir/data/data4/]]  heartbeating to lucene2-us-west.apache.org/127.0.0.1:37818) [    ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
   [junit4]   2> 2738649 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0xbd55e1bff1965: from storage DS-187a52ad-8bdf-40de-ac76-8a563f5862de node DatanodeRegistration(127.0.0.1:45222, datanodeUuid=45e13e34-ed33-4ba6-a068-8c26c0f895db, infoPort=37375, infoSecurePort=0, ipcPort=40226, storageInfo=lv=-56;cid=testClusterID;nsid=686266960;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
   [junit4]   2> 2738649 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0xbd55e1bff1965: from storage DS-65f3670a-709b-4d23-90f4-8fb68579d4ec node DatanodeRegistration(127.0.0.1:45222, datanodeUuid=45e13e34-ed33-4ba6-a068-8c26c0f895db, infoPort=37375, infoSecurePort=0, ipcPort=40226, storageInfo=lv=-56;cid=testClusterID;nsid=686266960;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 2738802 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 2738810 INFO  (Thread-2956) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 2738810 INFO  (Thread-2956) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 2738844 ERROR (Thread-2956) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 2738910 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.c.ZkTestServer start zk server on port:41867
   [junit4]   2> 2738949 INFO  (zkConnectionManagerCallback-2880-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2738988 INFO  (zkConnectionManagerCallback-2882-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2738995 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 2739007 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 2739012 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 2739022 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 2739023 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.c.AbstractZkTestCase put /h

[...truncated too long message...]

gistry=solr.core.collection1, tag=1159536118
   [junit4]   2> 7368177 INFO  (coreCloseExecutor-8580-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@7088c544: rootName = null, domain = solr.core.collection1, service url = null, agent id = null] for registry solr.core.collection1 / com.codahale.metrics.MetricRegistry@7773a06a
   [junit4]   2> 7368183 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[AF42AF945BE7BEEA]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@32b1b5bc{/solr,null,UNAVAILABLE}
   [junit4]   2> 7368183 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[AF42AF945BE7BEEA]) [    ] o.e.j.s.session Stopped scavenging
   [junit4]   2> 7368184 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[AF42AF945BE7BEEA]) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@6901ed42{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 7368184 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=332741528
   [junit4]   2> 7368185 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 7368185 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@50cda783: rootName = null, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@6e666a1d
   [junit4]   2> 7368201 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 7368201 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@16c1c0e0: rootName = null, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@2b09f016
   [junit4]   2> 7368205 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 7368205 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[AF42AF945BE7BEEA]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@37797d68: rootName = null, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@2e1b0b08
   [junit4]   2> 7368206 INFO  (coreCloseExecutor-8582-thread-1) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@525fe25f
   [junit4]   2> 7368206 INFO  (coreCloseExecutor-8582-thread-1) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=1382015583
   [junit4]   2> 7368206 INFO  (coreCloseExecutor-8582-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@6762e5a1: rootName = null, domain = solr.core.collection1, service url = null, agent id = null] for registry solr.core.collection1 / com.codahale.metrics.MetricRegistry@43417b78
   [junit4]   2> 7368216 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[AF42AF945BE7BEEA]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@22c49af1{/solr,null,UNAVAILABLE}
   [junit4]   2> 7368217 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[AF42AF945BE7BEEA]) [    ] o.e.j.s.session Stopped scavenging
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_AF42AF945BE7BEEA-001
   [junit4]   2> NOTE: test params are: codec=DummyCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=DUMMY, chunkSize=8, maxDocsPerChunk=3, blockSize=984), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=DUMMY, chunkSize=8, blockSize=984)), sim=RandomSimilarity(queryNorm=true): {}, locale=sr-Latn-RS, timezone=Australia/Perth
   [junit4]   2> NOTE: Linux 4.4.0-112-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=1,free=280582064,total=527433728
   [junit4]   2> NOTE: All tests run in this JVM: [DirectSolrConnectionTest, AtomicUpdateProcessorFactoryTest, SolrInfoBeanTest, BlockJoinFacetSimpleTest, AssignBackwardCompatibilityTest, JsonLoaderTest, CloudExitableDirectoryReaderTest, TestDynamicLoading, TestRTGBase, TestInitQParser, SpellCheckCollatorWithCollapseTest, CSVRequestHandlerTest, SuggesterTest, TestStressVersions, TestApiFramework, TestUseDocValuesAsStored2, ForceLeaderTest, V2ApiIntegrationTest, TestFunctionQuery, BaseCdcrDistributedZkTest, DistribCursorPagingTest, MetricUtilsTest, BitVectorTest, TestPerFieldSimilarity, BooleanFieldTest, InfoHandlerTest, TestLRUStatsCache, MetricsConfigTest, DistributedIntervalFacetingTest, HdfsAutoAddReplicasIntegrationTest, TestShardHandlerFactory, TestFieldCacheSortRandom, TestRecovery, PeerSyncTest, TestSimpleTrackingShardHandler, PrimitiveFieldTypeTest, OverseerTest, OverseerRolesTest, DeleteReplicaTest, FastVectorHighlighterTest, TestDynamicFieldResource, RequiredFieldsTest, CdcrReplicationDistributedZkTest, UUIDUpdateProcessorFallbackTest, DeleteInactiveReplicaTest, TestStressRecovery, AutoScalingHandlerTest, ZkStateWriterTest, QueryResultKeyTest, TestLeaderElectionWithEmptyReplica, TestSolrCLIRunExample, HighlighterTest, ConnectionManagerTest, TestSearchPerf, DirectoryFactoryTest, FieldMutatingUpdateProcessorTest, TestShortCircuitedRequests, SuggesterFSTTest, TestZkChroot, TestRetrieveFieldsOptimizer, TestFuzzyAnalyzedSuggestions, ResourceLoaderTest, BJQParserTest, TestComplexPhraseQParserPlugin, TestIntervalFaceting, QueryParsingTest, DistributedMLTComponentTest, TestCloudManagedSchema, TestDistributedGrouping, CoreAdminCreateDiscoverTest, TestConfigSetsAPI, TestMultiValuedNumericRangeQuery, ScriptEngineTest, TestInitParams, TestUninvertingReader, TestXIncludeConfig, TestBulkSchemaAPI, TestSolrCloudWithDelegationTokens, MergeStrategyTest, DistributedQueueTest, TestLegacyFieldCache, TestTrackingShardHandlerFactory, TestHdfsCloudBackupRestore, TestRequestStatusCollectionAPI, TestCloudJSONFacetJoinDomain, AssignTest, TestPostingsSolrHighlighter, CachingDirectoryFactoryTest, PreAnalyzedUpdateProcessorTest, XsltUpdateRequestHandlerTest, HdfsBasicDistributedZkTest, TestQueryUtils, SolrCloudExampleTest, PeerSyncReplicationTest, TestCharFilters, TestRestoreCore, TestPolicyCloud, DirectUpdateHandlerOptimizeTest, TestConfigSetImmutable, TestExpandComponent, TestRandomFlRTGCloud, PathHierarchyTokenizerFactoryTest, QueryEqualityTest, TestSolr4Spatial2, DocumentBuilderTest, TestDistribStateManager, TestNestedDocsSort, SolrGraphiteReporterTest, HdfsNNFailoverTest, TestElisionMultitermQuery, SoftAutoCommitTest, SuggestComponentContextFilterQueryTest, SolrCoreTest, HdfsDirectoryTest, TestEmbeddedSolrServerSchemaAPI, TestImpersonationWithHadoopAuth, QueryElevationComponentTest, TestReplicationHandler]
   [junit4] Completed [585/783 (6!)] on J1 in 310.78s, 16 tests, 1 error <<< FAILURES!

[...truncated 8159 lines...]
   [junit4] Suite: org.apache.solr.ltr.TestLTRReRankingPipeline
   [junit4]   2> 81344 INFO  (TEST-TestLTRReRankingPipeline.testDifferentTopN-seed#[44690DD45D5398]) [    ] o.a.s.l.TestLTRReRankingPipeline rerank 1 documents 
   [junit4]   2> 81359 INFO  (TEST-TestLTRReRankingPipeline.testDifferentTopN-seed#[44690DD45D5398]) [    ] o.a.s.l.TestLTRReRankingPipeline doc 0 in pos 0
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLTRReRankingPipeline -Dtests.method=testDifferentTopN -Dtests.seed=44690DD45D5398 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.badapples=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=ar-IQ -Dtests.timezone=AGT -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] FAILURE 0.47s J2 | TestLTRReRankingPipeline.testDifferentTopN <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: expected:<1.0> but was:<0.0>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([44690DD45D5398:F1E51B5DE1E6990A]:0)
   [junit4]    > 	at org.apache.solr.ltr.TestLTRReRankingPipeline.testDifferentTopN(TestLTRReRankingPipeline.java:255)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {field=FSTOrd50, id=Lucene50(blocksize=128), final-score=Lucene50(blocksize=128)}, docValues:{final-score=DocValuesFormat(name=Memory)}, maxPointsInLeafNode=1587, maxMBSortInHeap=7.418514316009276, sim=RandomSimilarity(queryNorm=true): {field=IB LL-L2}, locale=ar-IQ, timezone=AGT
   [junit4]   2> NOTE: Linux 4.4.0-112-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=1,free=248693496,total=310378496
   [junit4]   2> NOTE: All tests run in this JVM: [TestModelManager, TestUserTermScorerQuery, TestFeatureLogging, TestFilterSolrFeature, TestMultipleAdditiveTreesModel, TestRerankBase, TestLTRQParserExplain, TestLTRQParserPlugin, TestLTRReRankingPipeline]
   [junit4] Completed [20/37 (1!)] on J2 in 1.12s, 3 tests, 1 failure <<< FAILURES!

[...truncated 42067 lines...]