You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2017/10/12 23:31:38 UTC

[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1401 - Still unstable

Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1401/

8 tests failed.
FAILED:  org.apache.solr.cloud.TestLocalFSCloudBackupRestore.test

Error Message:
expected:<COMPLETED> but was:<RUNNING>

Stack Trace:
java.lang.AssertionError: expected:<COMPLETED> but was:<RUNNING>
	at __randomizedtesting.SeedInfo.seed([317B471B1312A108:B92F78C1BDEECCF0]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:147)
	at org.apache.solr.cloud.AbstractCloudBackupRestoreTestCase.testBackupAndRestore(AbstractCloudBackupRestoreTestCase.java:277)
	at org.apache.solr.cloud.AbstractCloudBackupRestoreTestCase.test(AbstractCloudBackupRestoreTestCase.java:136)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.TriLevelCompositeIdRoutingTest

Error Message:
1 thread leaked from SUITE scope at org.apache.solr.cloud.TriLevelCompositeIdRoutingTest:     1) Thread[id=6819, name=zkCallback-818-thread-1, state=TIMED_WAITING, group=TGRP-TriLevelCompositeIdRoutingTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.TriLevelCompositeIdRoutingTest: 
   1) Thread[id=6819, name=zkCallback-818-thread-1, state=TIMED_WAITING, group=TGRP-TriLevelCompositeIdRoutingTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([317B471B1312A108]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.TriLevelCompositeIdRoutingTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=6819, name=zkCallback-818-thread-1, state=TIMED_WAITING, group=TGRP-TriLevelCompositeIdRoutingTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=6819, name=zkCallback-818-thread-1, state=TIMED_WAITING, group=TGRP-TriLevelCompositeIdRoutingTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([317B471B1312A108]:0)


FAILED:  org.apache.solr.client.solrj.impl.CloudSolrClientTest.testOverwriteOption

Error Message:
Could not load collection from ZK: overwrite

Stack Trace:
org.apache.solr.common.SolrException: Could not load collection from ZK: overwrite
	at __randomizedtesting.SeedInfo.seed([F4F708FE41B56716:18CCE9152FE518C0]:0)
	at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1170)
	at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:690)
	at org.apache.solr.common.cloud.ClusterState.getCollectionOrNull(ClusterState.java:130)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:154)
	at org.apache.solr.client.solrj.impl.CloudSolrClientTest.testOverwriteOption(CloudSolrClientTest.java:155)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.junit.rules.ExpectedException$ExpectedExceptionStatement.evaluate(ExpectedException.java:110)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/overwrite/state.json
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
	at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212)
	at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:357)
	at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:354)
	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
	at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)
	at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1182)
	at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1168)
	... 44 more


FAILED:  org.apache.lucene.index.TestDuelingCodecsAtNight.testBigEquals

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
	at __randomizedtesting.SeedInfo.seed([687DBA3F681653F7]:0)


FAILED:  junit.framework.TestSuite.org.apache.lucene.index.TestDuelingCodecsAtNight

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
	at __randomizedtesting.SeedInfo.seed([687DBA3F681653F7]:0)


FAILED:  org.apache.solr.security.TestPKIAuthenticationPlugin.test

Error Message:


Stack Trace:
java.lang.NullPointerException
	at __randomizedtesting.SeedInfo.seed([317B471B1312A108:B92F78C1BDEECCF0]:0)
	at org.apache.solr.security.TestPKIAuthenticationPlugin.test(TestPKIAuthenticationPlugin.java:100)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.hdfs.HdfsTlogReplayBufferedWhileIndexingTest.test

Error Message:
There are still nodes recoverying - waited for 600 seconds

Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 600 seconds
	at __randomizedtesting.SeedInfo.seed([317B471B1312A108:B92F78C1BDEECCF0]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:185)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:914)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1471)
	at org.apache.solr.cloud.TlogReplayBufferedWhileIndexingTest.test(TlogReplayBufferedWhileIndexingTest.java:99)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 2201 lines...]
   [junit4] Suite: org.apache.lucene.index.TestDuelingCodecsAtNight
   [junit4]   2> ??? 12, 2017 2:19:53 PM com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
   [junit4]   2> WARNING: Suite execution timed out: org.apache.lucene.index.TestDuelingCodecsAtNight
   [junit4]   2>    1) Thread[id=5590, name=SUITE-TestDuelingCodecsAtNight-seed#[687DBA3F681653F7], state=RUNNABLE, group=TGRP-TestDuelingCodecsAtNight]
   [junit4]   2>         at java.lang.Thread.getStackTrace(Thread.java:1559)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:696)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:693)
   [junit4]   2>         at java.security.AccessController.doPrivileged(Native Method)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getStackTrace(ThreadLeakControl.java:693)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getThreadsWithTraces(ThreadLeakControl.java:709)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.formatThreadStacksFull(ThreadLeakControl.java:689)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.access$1000(ThreadLeakControl.java:65)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$2.evaluate(ThreadLeakControl.java:415)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:705)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.access$200(RandomizedRunner.java:139)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$2.run(RandomizedRunner.java:626)
   [junit4]   2>    2) Thread[id=1, name=main, state=WAITING, group=main]
   [junit4]   2>         at java.lang.Object.wait(Native Method)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1252)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1326)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:636)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:493)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:251)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
   [junit4]   2>    3) Thread[id=5591, name=TEST-TestDuelingCodecsAtNight.testBigEquals-seed#[687DBA3F681653F7], state=RUNNABLE, group=TGRP-TestDuelingCodecsAtNight]
   [junit4]   2>         at org.apache.lucene.store.MockIndexInputWrapper.readByte(MockIndexInputWrapper.java:140)
   [junit4]   2>         at org.apache.lucene.codecs.simpletext.SimpleTextUtil.readLine(SimpleTextUtil.java:59)
   [junit4]   2>         at org.apache.lucene.codecs.simpletext.SimpleTextFieldsReader$SimpleTextPostingsEnum.nextDoc(SimpleTextFieldsReader.java:409)
   [junit4]   2>         at org.apache.lucene.search.DocIdSetIterator.slowAdvance(DocIdSetIterator.java:165)
   [junit4]   2>         at org.apache.lucene.codecs.simpletext.SimpleTextFieldsReader$SimpleTextPostingsEnum.advance(SimpleTextFieldsReader.java:448)
   [junit4]   2>         at org.apache.lucene.index.CheckIndex.testTermVectors(CheckIndex.java:2475)
   [junit4]   2>         at org.apache.lucene.index.CheckIndex.checkIndex(CheckIndex.java:753)
   [junit4]   2>         at org.apache.lucene.util.TestUtil.checkIndex(TestUtil.java:300)
   [junit4]   2>         at org.apache.lucene.store.MockDirectoryWrapper.close(MockDirectoryWrapper.java:865)
   [junit4]   2>         at org.apache.lucene.util.IOUtils.close(IOUtils.java:89)
   [junit4]   2>         at org.apache.lucene.util.IOUtils.close(IOUtils.java:76)
   [junit4]   2>         at org.apache.lucene.index.TestDuelingCodecs.tearDown(TestDuelingCodecs.java:98)
   [junit4]   2>         at sun.reflect.GeneratedMethodAccessor13.invoke(Unknown Source)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2>    4) Thread[id=9, name=JUnit4-serializer-daemon, state=TIMED_WAITING, group=main]
   [junit4]   2>         at java.lang.Thread.sleep(Native Method)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:50)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestDuelingCodecsAtNight -Dtests.method=testBigEquals -Dtests.seed=687DBA3F681653F7 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=sr-RS -Dtests.timezone=Africa/Dakar -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   7201s J1 | TestDuelingCodecsAtNight.testBigEquals <<<
   [junit4]    > Throwable #1: java.lang.Exception: Test abandoned because suite timeout was reached.
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([687DBA3F681653F7]:0)
   [junit4]   2> ??? 12, 2017 2:19:53 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> ??? 12, 2017 2:20:13 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 1 thread leaked from SUITE scope at org.apache.lucene.index.TestDuelingCodecsAtNight: 
   [junit4]   2>    1) Thread[id=5591, name=TEST-TestDuelingCodecsAtNight.testBigEquals-seed#[687DBA3F681653F7], state=RUNNABLE, group=TGRP-TestDuelingCodecsAtNight]
   [junit4]   2>         at org.apache.lucene.store.MockIndexInputWrapper.readByte(MockIndexInputWrapper.java:140)
   [junit4]   2>         at org.apache.lucene.codecs.simpletext.SimpleTextUtil.readLine(SimpleTextUtil.java:59)
   [junit4]   2>         at org.apache.lucene.codecs.simpletext.SimpleTextFieldsReader$SimpleTextPostingsEnum.nextDoc(SimpleTextFieldsReader.java:409)
   [junit4]   2>         at org.apache.lucene.search.DocIdSetIterator.slowAdvance(DocIdSetIterator.java:165)
   [junit4]   2>         at org.apache.lucene.codecs.simpletext.SimpleTextFieldsReader$SimpleTextPostingsEnum.advance(SimpleTextFieldsReader.java:448)
   [junit4]   2>         at org.apache.lucene.index.CheckIndex.testTermVectors(CheckIndex.java:2475)
   [junit4]   2>         at org.apache.lucene.index.CheckIndex.checkIndex(CheckIndex.java:753)
   [junit4]   2>         at org.apache.lucene.util.TestUtil.checkIndex(TestUtil.java:300)
   [junit4]   2>         at org.apache.lucene.store.MockDirectoryWrapper.close(MockDirectoryWrapper.java:865)
   [junit4]   2>         at org.apache.lucene.util.IOUtils.close(IOUtils.java:89)
   [junit4]   2>         at org.apache.lucene.util.IOUtils.close(IOUtils.java:76)
   [junit4]   2>         at org.apache.lucene.index.TestDuelingCodecs.tearDown(TestDuelingCodecs.java:98)
   [junit4]   2>         at sun.reflect.GeneratedMethodAccessor13.invoke(Unknown Source)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> ??? 12, 2017 2:20:13 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=5591, name=TEST-TestDuelingCodecsAtNight.testBigEquals-seed#[687DBA3F681653F7], state=RUNNABLE, group=TGRP-TestDuelingCodecsAtNight]
   [junit4]   2> ??? 12, 2017 2:20:16 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> SEVERE: There are still zombie threads that couldn't be terminated:
   [junit4]   2>    1) Thread[id=5591, name=TEST-TestDuelingCodecsAtNight.testBigEquals-seed#[687DBA3F681653F7], state=RUNNABLE, group=TGRP-TestDuelingCodecsAtNight]
   [junit4]   2>         at org.apache.lucene.store.MockIndexInputWrapper.getFilePointer(MockIndexInputWrapper.java:122)
   [junit4]   2>         at org.apache.lucene.codecs.simpletext.SimpleTextFieldsReader$SimpleTextPostingsEnum.nextDoc(SimpleTextFieldsReader.java:408)
   [junit4]   2>         at org.apache.lucene.search.DocIdSetIterator.slowAdvance(DocIdSetIterator.java:165)
   [junit4]   2>         at org.apache.lucene.codecs.simpletext.SimpleTextFieldsReader$SimpleTextPostingsEnum.advance(SimpleTextFieldsReader.java:448)
   [junit4]   2>         at org.apache.lucene.index.CheckIndex.testTermVectors(CheckIndex.java:2475)
   [junit4]   2>         at org.apache.lucene.index.CheckIndex.checkIndex(CheckIndex.java:753)
   [junit4]   2>         at org.apache.lucene.util.TestUtil.checkIndex(TestUtil.java:300)
   [junit4]   2>         at org.apache.lucene.store.MockDirectoryWrapper.close(MockDirectoryWrapper.java:865)
   [junit4]   2>         at org.apache.lucene.util.IOUtils.close(IOUtils.java:89)
   [junit4]   2>         at org.apache.lucene.util.IOUtils.close(IOUtils.java:76)
   [junit4]   2>         at org.apache.lucene.index.TestDuelingCodecs.tearDown(TestDuelingCodecs.java:98)
   [junit4]   2>         at sun.reflect.GeneratedMethodAccessor13.invoke(Unknown Source)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=419, maxMBSortInHeap=7.163448164681653, sim=RandomSimilarity(queryNorm=true): {titleTokenized=DFR GLZ(0.3), body=DFR I(n)L3(800.0)}, locale=sr-RS, timezone=Africa/Dakar
   [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=2,free=247517408,total=530055168
   [junit4]   2> NOTE: All tests run in this JVM: [TestBytesRefAttImpl, TestIndexWriterOutOfFileDescriptors, TestNumericDocValuesUpdates, TestLucene50StoredFieldsFormat, Test2BDocs, TestPrefixInBooleanQuery, TestSpanExplanations, TestConcurrentMergeScheduler, TestOperations, TestFilterIterator, TestRAMDirectory, TestRollback, TestBKD, TestMaxPosition, TestPhrasePrefixQuery, TestRecyclingIntBlockAllocator, TestTimeLimitingCollector, TestPolygon, TestTragicIndexWriterDeadlock, TestTermVectorsReader, TestPhraseQuery, Test2BPostingsBytes, TestNativeFSLockFactory, TestGrowableByteArrayDataOutput, TestSameScoresWithThreads, TestIndexWriterUnicode, TestReusableStringReader, TestDocValuesQueries, TestTrackingDirectoryWrapper, TestDeterminism, TestNorms, TestSpansEnum, TestCheckIndex, TestLockFactory, TestFieldValueQuery, TestSearcherManager, TestLongBitSet, TestOmitPositions, TestRecyclingByteBlockAllocator, TestTransactions, TestScoreCachingWrappingScorer, FiniteStringsIteratorTest, TestFieldInvertState, TestSpanCollection, TestIndexWriterOnJRECrash, TestFileSwitchDirectory, TestGraphTokenizers, TestMinimize, TestSortedNumericSortField, TestFilterDirectoryReader, TestCloseableThreadLocal, TestForTooMuchCloning, TestPointValues, TestClassicSimilarity, TestGraphTokenStreamFiniteStrings, TestSynonymQuery, TestMultiPhraseEnum, TestCodecUtil, TestToken, TestAddIndexes, TestParallelReaderEmptyIndex, TestTieredMergePolicy, TestVirtualMethod, TestSegmentInfos, TestOfflineSorter, TestSimpleFSLockFactory, TestParallelLeafReader, TestDocIdSetBuilder, TestAxiomaticSimilarity, TestIndexFileDeleter, TestByteSlices, TestComplexExplanations, TestSpanNotQuery, TestDocumentsWriterDeleteQueue, TestDemo, TestCharArraySet, TestIndexWriterMerging, TestFSTs, TestMultiMMap, TestBooleanOr, TestBasics, TestIndexWriterDelete, TestIndexWriterMergePolicy, TestDuelingCodecs, TestBoolean2, TestAtomicUpdate, TestStressAdvance, TestDirectoryReaderReopen, TestSearchWithThreads, TestSearchAfter, TestHighCompressionMode, TestDocumentsWriterStallControl, TestArrayUtil, TestLevenshteinAutomata, TestBufferedIndexInput, TestTermsEnum, TestCodecHoldsOpenFiles, TestReaderClosed, TestNGramPhraseQuery, TestTermdocPerf, TestIndexCommit, TestTerm, TestPositiveScoresOnlyCollector, TestNewestSegment, TestByteArrayDataInput, Test2BNumericDocValues, TestCharArrayMap, TestStopFilter, TestBlockPostingsFormat, TestBlockPostingsFormat3, TestForUtil, TestLucene50CompoundFormat, TestLucene50TermVectorsFormat, TestLucene60FieldInfoFormat, TestLucene60PointsFormat, TestLucene70DocValuesFormat, TestLucene70SegmentInfoFormat, TestFieldType, TestGeoEncodingUtils, TestGeoUtils, TestPolygon2D, Test2BSortedDocValuesFixedSorted, TestAllFilesDetectTruncation, TestAllFilesHaveChecksumFooter, TestCustomTermFreq, TestDocIDMerger, TestDocInverterPerFieldErrorInfo, TestDuelingCodecsAtNight]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestDuelingCodecsAtNight -Dtests.seed=687DBA3F681653F7 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=sr-RS -Dtests.timezone=Africa/Dakar -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   0.00s J1 | TestDuelingCodecsAtNight (suite) <<<
   [junit4]    > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([687DBA3F681653F7]:0)
   [junit4] Completed [409/455 (1!)] on J1 in 7224.88s, 1 test, 2 errors <<< FAILURES!

[...truncated 10644 lines...]
   [junit4] Suite: org.apache.solr.security.TestPKIAuthenticationPlugin
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.security.TestPKIAuthenticationPlugin_317B471B1312A108-001/init-core-data-001
   [junit4]   2> 181512 WARN  (SUITE-TestPKIAuthenticationPlugin-seed#[317B471B1312A108]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=9 numCloses=9
   [junit4]   2> 181512 INFO  (SUITE-TestPKIAuthenticationPlugin-seed#[317B471B1312A108]-worker) [    ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 181514 INFO  (SUITE-TestPKIAuthenticationPlugin-seed#[317B471B1312A108]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 181514 INFO  (SUITE-TestPKIAuthenticationPlugin-seed#[317B471B1312A108]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 181565 INFO  (TEST-TestPKIAuthenticationPlugin.test-seed#[317B471B1312A108]) [    ] o.a.s.SolrTestCaseJ4 ###Starting test
   [junit4]   2> 195496 ERROR (TEST-TestPKIAuthenticationPlugin.test-seed#[317B471B1312A108]) [    ] o.a.s.s.PKIAuthenticationPlugin Invalid key request timestamp: 1507839220544 , received timestamp: 1507839233359 , TTL: 10000
   [junit4]   2> 195497 INFO  (TEST-TestPKIAuthenticationPlugin.test-seed#[317B471B1312A108]) [    ] o.a.s.SolrTestCaseJ4 ###Ending test
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestPKIAuthenticationPlugin -Dtests.method=test -Dtests.seed=317B471B1312A108 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=ar-LY -Dtests.timezone=Europe/Simferopol -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   14.0s J0 | TestPKIAuthenticationPlugin.test <<<
   [junit4]    > Throwable #1: java.lang.NullPointerException
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([317B471B1312A108:B92F78C1BDEECCF0]:0)
   [junit4]    > 	at org.apache.solr.security.TestPKIAuthenticationPlugin.test(TestPKIAuthenticationPlugin.java:100)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.security.TestPKIAuthenticationPlugin_317B471B1312A108-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=1661, maxMBSortInHeap=7.1266937688702345, sim=RandomSimilarity(queryNorm=false): {}, locale=ar-LY, timezone=Europe/Simferopol
   [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=350441992,total=430964736
   [junit4]   2> NOTE: All tests run in this JVM: [TestInitQParser, TestDocSet, TestBulkSchemaAPI, TestPKIAuthenticationPlugin]
   [junit4] Completed [10/745 (1!)] on J0 in 14.35s, 1 test, 1 error <<< FAILURES!

[...truncated 551 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestLocalFSCloudBackupRestore
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_317B471B1312A108-001/init-core-data-001
   [junit4]   2> 3299920 WARN  (SUITE-TestLocalFSCloudBackupRestore-seed#[317B471B1312A108]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=13 numCloses=13
   [junit4]   2> 3299920 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[317B471B1312A108]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 3299921 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[317B471B1312A108]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 3299921 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[317B471B1312A108]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 3299956 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[317B471B1312A108]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_317B471B1312A108-001/tempDir-001
   [junit4]   2> 3299957 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[317B471B1312A108]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 3300141 INFO  (Thread-3150) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 3300141 INFO  (Thread-3150) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 3300141 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[317B471B1312A108]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:39230
   [junit4]   2> 3300142 ERROR (Thread-3150) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 3300256 INFO  (jetty-launcher-1118-thread-1) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 3300372 INFO  (jetty-launcher-1118-thread-2) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 3300373 INFO  (jetty-launcher-1118-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@16e77be4{/solr,null,AVAILABLE}
   [junit4]   2> 3300374 INFO  (jetty-launcher-1118-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@1f4721df{HTTP/1.1,[http/1.1]}{127.0.0.1:47294}
   [junit4]   2> 3300374 INFO  (jetty-launcher-1118-thread-2) [    ] o.e.j.s.Server Started @3319957ms
   [junit4]   2> 3300374 INFO  (jetty-launcher-1118-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=47294}
   [junit4]   2> 3300374 ERROR (jetty-launcher-1118-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 3300374 INFO  (jetty-launcher-1118-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 3300374 INFO  (jetty-launcher-1118-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 3300374 INFO  (jetty-launcher-1118-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 3300374 INFO  (jetty-launcher-1118-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-12T21:05:37.238Z
   [junit4]   2> 3300376 INFO  (jetty-launcher-1118-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@486f5f7e{/solr,null,AVAILABLE}
   [junit4]   2> 3300376 INFO  (jetty-launcher-1118-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@4ddefc04{HTTP/1.1,[http/1.1]}{127.0.0.1:57269}
   [junit4]   2> 3300376 INFO  (jetty-launcher-1118-thread-1) [    ] o.e.j.s.Server Started @3319960ms
   [junit4]   2> 3300465 INFO  (jetty-launcher-1118-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=57269}
   [junit4]   2> 3300465 ERROR (jetty-launcher-1118-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 3300465 INFO  (jetty-launcher-1118-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 3300465 INFO  (jetty-launcher-1118-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 3300465 INFO  (jetty-launcher-1118-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 3300465 INFO  (jetty-launcher-1118-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-12T21:05:37.329Z
   [junit4]   2> 3300466 INFO  (jetty-launcher-1118-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 3300492 INFO  (jetty-launcher-1118-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 3300529 INFO  (jetty-launcher-1118-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39230/solr
   [junit4]   2> 3300547 INFO  (jetty-launcher-1118-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39230/solr
   [junit4]   2> 3300746 INFO  (jetty-launcher-1118-thread-1) [n:127.0.0.1:57269_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 3300747 INFO  (jetty-launcher-1118-thread-1) [n:127.0.0.1:57269_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:57269_solr
   [junit4]   2> 3300753 INFO  (jetty-launcher-1118-thread-1) [n:127.0.0.1:57269_solr    ] o.a.s.c.Overseer Overseer (id=98817955398025222-127.0.0.1:57269_solr-n_0000000000) starting
   [junit4]   2> 3300882 INFO  (jetty-launcher-1118-thread-1) [n:127.0.0.1:57269_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57269_solr
   [junit4]   2> 3300896 INFO  (zkCallback-1130-thread-1-processing-n:127.0.0.1:57269_solr) [n:127.0.0.1:57269_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 3301253 INFO  (jetty-launcher-1118-thread-2) [n:127.0.0.1:47294_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 3301254 INFO  (jetty-launcher-1118-thread-2) [n:127.0.0.1:47294_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 3301381 INFO  (jetty-launcher-1118-thread-2) [n:127.0.0.1:47294_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:47294_solr
   [junit4]   2> 3301457 INFO  (zkCallback-1130-thread-1-processing-n:127.0.0.1:57269_solr) [n:127.0.0.1:57269_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 3301471 INFO  (zkCallback-1129-thread-1-processing-n:127.0.0.1:47294_solr) [n:127.0.0.1:47294_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 3302098 INFO  (jetty-launcher-1118-thread-2) [n:127.0.0.1:47294_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_47294.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@47e6ecf2
   [junit4]   2> 3302180 INFO  (jetty-launcher-1118-thread-2) [n:127.0.0.1:47294_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_47294.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@47e6ecf2
   [junit4]   2> 3302180 INFO  (jetty-launcher-1118-thread-2) [n:127.0.0.1:47294_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_47294.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@47e6ecf2
   [junit4]   2> 3302237 INFO  (jetty-launcher-1118-thread-2) [n:127.0.0.1:47294_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_317B471B1312A108-001/tempDir-001/node2/.
   [junit4]   2> 3302962 INFO  (jetty-launcher-1118-thread-1) [n:127.0.0.1:57269_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_57269.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@47e6ecf2
   [junit4]   2> 3303027 INFO  (jetty-launcher-1118-thread-1) [n:127.0.0.1:57269_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_57269.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@47e6ecf2
   [junit4]   2> 3303028 INFO  (jetty-launcher-1118-thread-1) [n:127.0.0.1:57269_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_57269.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@47e6ecf2
   [junit4]   2> 3303049 INFO  (jetty-launcher-1118-thread-1) [n:127.0.0.1:57269_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_317B471B1312A108-001/tempDir-001/node1/.
   [junit4]   2> 3303233 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[317B471B1312A108]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 3303234 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[317B471B1312A108]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:39230/solr ready
   [junit4]   2> 3304021 INFO  (TEST-TestLocalFSCloudBackupRestore.test-seed#[317B471B1312A108]) [    ] o.a.s.SolrTestCaseJ4 ###Starting test
   [junit4]   2> 3304025 INFO  (qtp1531139025-8209) [n:127.0.0.1:47294_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params pullReplicas=1&replicationFactor=2&property.customKey=customValue&collection.configName=conf1&maxShardsPerNode=8&name=backuprestore&nrtReplicas=2&action=CREATE&numShards=2&tlogReplicas=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 3304027 INFO  (OverseerThreadFactory-2370-thread-1-processing-n:127.0.0.1:57269_solr) [n:127.0.0.1:57269_solr    ] o.a.s.c.CreateCollectionCmd Create collection backuprestore
   [junit4]   2> 3304045 WARN  (OverseerThreadFactory-2370-thread-1-processing-n:127.0.0.1:57269_solr) [n:127.0.0.1:57269_solr    ] o.a.s.c.CreateCollectionCmd Specified number of replicas of 4 on collection backuprestore is higher than the number of Solr instances currently live or live and part of your createNodeSet(2). It's unusual to run two replica of the same slice on the same Solr-instance.
   [junit4]   2> 3304183 INFO  (OverseerStateUpdate-98817955398025222-127.0.0.1:57269_solr-n_0000000000) [n:127.0.0.1:57269_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"backuprestore",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"backuprestore_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:47294/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 3304201 INFO  (OverseerStateUpdate-98817955398025222-127.0.0.1:57269_solr-n_0000000000) [n:127.0.0.1:57269_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"backuprestore",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"backuprestore_shard1_replica_n3",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:57269/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 3304204 INFO  (OverseerStateUpdate-98817955398025222-127.0.0.1:57269_solr-n_0000000000) [n:127.0.0.1:57269_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"backuprestore",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"backuprestore_shard1_replica_t5",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:47294/solr",
   [junit4]   2>   "type":"TLOG"} 
   [junit4]   2> 3304354 INFO  (OverseerStateUpdate-98817955398025222-127.0.0.1:57269_solr-n_0000000000) [n:127.0.0.1:57269_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"backuprestore",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"backuprestore_shard1_replica_p7",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:57269/solr",
   [junit4]   2>   "type":"PULL"} 
   [junit4]   2> 3304359 INFO  (OverseerStateUpdate-98817955398025222-127.0.0.1:57269_solr-n_0000000000) [n:127.0.0.1:57269_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"backuprestore",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"backuprestore_shard2_replica_n9",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:47294/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 3304361 INFO  (OverseerStateUpdate-98817955398025222-127.0.0.1:57269_solr-n_0000000000) [n:127.0.0.1:57269_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"backuprestore",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"backuprestore_shard2_replica_n11",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:57269/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 3304364 INFO  (OverseerStateUpdate-98817955398025222-127.0.0.1:57269_solr-n_0000000000) [n:127.0.0.1:57269_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"backuprestore",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"backuprestore_shard2_replica_t13",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:47294/solr",
   [junit4]   2>   "type":"TLOG"} 
   [junit4]   2> 3304366 INFO  (OverseerStateUpdate-98817955398025222-127.0.0.1:57269_solr-n_0000000000) [n:127.0.0.1:57269_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"backuprestore",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"backuprestore_shard2_replica_p14",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:57269/solr",
   [junit4]   2>   "type":"PULL"} 
   [junit4]   2> 3304539 INFO  (qtp1531139025-8207) [n:127.0.0.1:47294_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=NRT&property.customKey=customValue&coreNodeName=core_node2&name=backuprestore_shard1_replica_n1&action=CREATE&numShards=2&shard=shard1&wt=javabin
   [junit4]   2> 3304539 INFO  (qtp1531139025-8207) [n:127.0.0.1:47294_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 3304885 INFO  (qtp229150333-8218) [n:127.0.0.1:57269_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=PULL&property.customKey=customValue&coreNodeName=core_node8&name=backuprestore_shard1_replica_p7&action=CREATE&numShards=2&shard=shard1&wt=javabin
   [junit4]   2> 3304885 INFO  (qtp1531139025-8211) [n:127.0.0.1:47294_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=TLOG&property.customKey=customValue&coreNodeName=core_node15&name=backuprestore_shard2_replica_t13&action=CREATE&numShards=2&shard=shard2&wt=javabin
   [junit4]   2> 3304885 INFO  (qtp229150333-8218) [n:127.0.0.1:57269_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 3304886 INFO  (qtp229150333-8217) [n:127.0.0.1:57269_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=NRT&property.customKey=customValue&coreNodeName=core_node4&name=backuprestore_shard1_replica_n3&action=CREATE&numShards=2&shard=shard1&wt=javabin
   [junit4]   2> 3304886 INFO  (qtp1531139025-8213) [n:127.0.0.1:47294_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=TLOG&property.customKey=customValue&coreNodeName=core_node6&name=backuprestore_shard1_replica_t5&action=CREATE&numShards=2&shard=shard1&wt=javabin
   [junit4]   2> 3304886 INFO  (qtp1531139025-8206) [n:127.0.0.1:47294_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=NRT&property.customKey=customValue&coreNodeName=core_node10&name=backuprestore_shard2_replica_n9&action=CREATE&numShards=2&shard=shard2&wt=javabin
   [junit4]   2> 3304889 INFO  (qtp229150333-8221) [n:127.0.0.1:57269_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=PULL&property.customKey=customValue&coreNodeName=core_node16&name=backuprestore_shard2_replica_p14&action=CREATE&numShards=2&shard=shard2&wt=javabin
   [junit4]   2> 3305034 INFO  (qtp229150333-8220) [n:127.0.0.1:57269_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=NRT&property.customKey=customValue&coreNodeName=core_node12&name=backuprestore_shard2_replica_n11&action=CREATE&numShards=2&shard=shard2&wt=javabin
   [junit4]   2> 3305161 INFO  (zkCallback-1129-thread-1-processing-n:127.0.0.1:47294_solr) [n:127.0.0.1:47294_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore/state.json] for collection [backuprestore] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 3305162 INFO  (zkCallback-1130-thread-2-processing-n:127.0.0.1:57269_solr) [n:127.0.0.1:57269_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore/state.json] for collection [backuprestore] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 3306028 INFO  (qtp1531139025-8213) [n:127.0.0.1:47294_solr c:backuprestore s:shard1 r:core_node6 x:backuprestore_shard1_replica_t5] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 3306042 INFO  (qtp1531139025-8206) [n:127.0.0.1:47294_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n9] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 3306058 INFO  (qtp229150333-8217) [n:127.0.0.1:57269_solr c:backuprestore s:shard1 r:core_node4 x:backuprestore_shard1_replica_n3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 3306077 INFO  (qtp1531139025-8211) [n:127.0.0.1:47294_solr c:backuprestore s:shard2 r:core_node15 x:backuprestore_shard2_replica_t13] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 3306084 INFO  (qtp229150333-8221) [n:127.0.0.1:57269_solr c:backuprestore s:shard2 r:core_node16 x:backuprestore_shard2_replica_p14] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 3306132 INFO  (qtp229150333-8217) [n:127.0.0.1:57269_solr c:backuprestore s:shard1 r:core_node4 x:backuprestore_shard1_replica_n3] o.a.s.s.IndexSchema [backuprestore_shard1_replica_n3] Schema name=minimal
   [junit4]   2> 3306271 INFO  (qtp229150333-8217) [n:127.0.0.1:57269_solr c:backuprestore s:shard1 r:core_node4 x:backuprestore_shard1_replica_n3] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 3306271 INFO  (qtp229150333-8217) [n:127.0.0.1:57269_solr c:backuprestore s:shard1 r:core_node4 x:backuprestore_shard1_replica_n3] o.a.s.c.CoreContainer Creating SolrCore 'backuprestore_shard1_replica_n3' using configuration from collection backuprestore, trusted=true
   [junit4]   2> 3306272 INFO  (qtp229150333-8217) [n:127.0.0.1:57269_solr c:backuprestore s:shard1 r:core_node4 x:backuprestore_shard1_replica_n3] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_57269.solr.core.backuprestore.shard1.replica_n3' (registry 'solr.core.backuprestore.shard1.replica_n3') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@47e6ecf2
   [junit4]   2> 3306272 INFO  (qtp229150333-8217) [n:127.0.0.1:57269_solr c:backuprestore s:shard1 r:core_node4 x:backuprestore_shard1_replica_n3] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 3306272 INFO  (qtp229150333-8217) [n:127.0.0.1:57269_solr c:backuprestore s:shard1 r:core_node4 x:backuprestore_shard1_replica_n3] o.a.s.c.SolrCore [[backuprestore_shard1_replica_n3] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_317B471B1312A108-001/tempDir-001/node1/backuprestore_shard1_replica_n3], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_317B471B1312A108-001/tempDir-001/node1/./backuprestore_shard1_replica_n3/data/]
   [junit4]   2> 3306289 INFO  (qtp229150333-8220) [n:127.0.0.1:57269_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_n11] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 3306292 INFO  (qtp229150333-8218) [n:127.0.0.1:57269_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_p7] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 3306314 INFO  (qtp229150333-8221) [n:127.0.0.1:57269_solr c:backuprestore s:shard2 r:core_node16 x:backuprestore_shard2_replica_p14] o.a.s.s.IndexSchema [backuprestore_shard2_replica_p14] Schema name=minimal
   [junit4]   2> 3306316 INFO  (qtp229150333-8221) [n:127.0.0.1:57269_solr c:backuprestore s:shard2 r:core_node16 x:backuprestore_shard2_replica_p14] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 3306316 INFO  (qtp229150333-8221) [n:127.0.0.1:57269_solr c:backuprestore s:shard2 r:core_node16 x:backuprestore_shard2_replica_p14] o.a.s.c.CoreContainer Creating SolrCore 'backuprestore_shard2_replica_p14' using configuration from collection backuprestore, trusted=true
   [junit4]   2> 3306332 INFO  (qtp1531139025-8211) [n:127.0.0.1:47294_solr c:backuprestore s:shard2 r:core_node15 x:backuprestore_shard2_replica_t13] o.a.s.s.IndexSchema [backuprestore_shard2_replica_t13] Schema name=minimal
   [junit4]   2> 3306353 INFO  (qtp1531139025-8213) [n:127.0.0.1:47294_solr c:backuprestore s:shard1 r:core_node6 x:backuprestore_shard1_replica_t5] o.a.s.s.IndexSchema [backuprestore_shard1_replica_t5] Schema name=minimal
   [junit4]   2> 3306355 INFO  (qtp1531139025-8213) [n:127.0.0.1:47294_solr c:backuprestore s:shard1 r:core_node6 x:backuprestore_shard1_replica_t5] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 3306355 INFO  (qtp1531139025-8213) [n:127.0.0.1:47294_solr c:backuprestore s:shard1 r:core_node6 x:backuprestore_shard1_replica_t5] o.a.s.c.CoreContainer Creating SolrCore 'backuprestore_shard1_replica_t5' using configuration from collection backuprestore, trusted=true
   [junit4]   2> 3306356 INFO  (qtp1531139025-8213) [n:127.0.0.1:47294_solr c:backuprestore s:shard1 r:core_node6 x:backuprestore_shard1_replica_t5] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_47294.solr.core.backuprestore.shard1.replica_t5' (registry 'solr.core.backuprestore.shard1.replica_t5') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@47e6ecf2
   [junit4]   2> 3306356 INFO  (qtp1531139025-8213) [n:127.0.0.1:47294_solr c:backuprestore s:shard1 r:core_node6 x:backuprestore_shard1_replica_t5] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 3306356 INFO  (qtp1531139025-8213) [n:127.0.0.1:47294_solr c:backuprestore s:shard1 r:core_node6 x:backuprestore_shard1_replica_t5] o.a.s.c.SolrCore [[backuprestore_shard1_replica_t5] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_317B471B1312A108-001/tempDir-001/node2/backuprestore_shard1_replica_t5], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_317B471B1312A108-001/tempDir-001/node2/./backuprestore_shard1_replica_t5/data/]
   [junit4]   2> 3306158 INFO  (qtp1531139025-8206) [n:127.0.0.1:47294_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n9] o.a.s.s.IndexSchema [backuprestore_shard2_replica_n9] Schema name=minimal
   [junit4]   2> 3306139 INFO  (qtp1531139025-8207) [n:127.0.0.1:47294_solr c:backuprestore s:shard1 r:core_node2 x:backuprestore_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 3306385 INFO  (qtp229150333-8221) [n:127.0.0.1:57269_solr c:backuprestore s:shard2 r:core_node16 x:backuprestore_shard2_replica_p14] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_57269.solr.core.backuprestore.shard2.replica_p14' (registry 'solr.core.backuprestore.shard2.replica_p14') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@47e6ecf2
   [junit4]   2> 3306385 INFO  (qtp229150333-8221) [n:127.0.0.1:57269_solr c:backuprestore s:shard2 r:core_node16 x:backuprestore_shard2_replica_p14] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 3306385 INFO  (qtp229150333-8221) [n:127.0.0.1:57269_solr c:backuprestore s:shard2 r:core_node16 x:backuprestore_shard2_replica_p14] o.a.s.c.SolrCore [[backuprestore_shard2_replica_p14] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_317B471B1312A108-001/tempDir-001/node1/backuprestore_shard2_replica_p14], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_317B471B1312A108-001/tempDir-001/node1/./backuprestore_shard2_replica_p14/data/]
   [junit4]   2> 3306387 INFO  (qtp1531139025-8206) [n:127.0.0.1:47294_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n9] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 3306387 INFO  (qtp1531139025-8206) [n:127.0.0.1:47294_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n9] o.a.s.c.CoreContainer Creating SolrCore 'backuprestore_shard2_replica_n9' using configuration from collection backuprestore, trusted=true
   [junit4]   2> 3306387 INFO  (qtp1531139025-8206) [n:127.0.0.1:47294_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n9] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_47294.solr.core.backuprestore.shard2.replica_n9' (registry 'solr.core.backuprestore.shard2.replica_n9') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@47e6ecf2
   [junit4]   2> 3306387 INFO  (qtp1531139025-8206) [n:127.0.0.1:47294_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n9] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 3306387 INFO  (qtp1531139025-8206) [n:127.0.0.1:47294_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n9] o.a.s.c.SolrCore [[backuprestore_shard2_replica_n9] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_317B471B1312A108-001/tempDir-001/node2/backuprestore_shard2_replica_n9], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_317B471B1312A108-001/tempDir-001/node2/./backuprestore_shard2_replica_n9/data/]
   [junit4]   2> 3306412 INFO  (qtp1531139025-8211) [n:127.0.0.1:47294_solr c:backuprestore s:shard2 r:core_node15 x:backuprestore_shard2_replica_t13] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 3306412 INFO  (qtp1531139025-8211) [n:127.0.0.1:47294_solr c:backuprestore s:shard2 r:core_node15 x:backuprestore_shard2_replica_t13] o.a.s.c.CoreContainer Creating SolrCore 'backuprestore_shard2_replica_t13' using configuration from collection backuprestore, trusted=true
   [junit4]   2> 3306415 INFO  (qtp229150333-8220) [n:127.0.0.1:57269_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_n11] o.a.s.s.IndexSchema [backuprestore_shard2_replica_n11] Schema name=minimal
   [junit4]   2> 3306435 INFO  (qtp229150333-8218) [n:127.0.0.1:57269_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_p7] o.a.s.s.IndexSchema [backuprestore_shard1_replica_p7] Schema name=minimal
   [junit4]   2> 3306454 INFO  (qtp1531139025-8207) [n:127.0.0.1:47294_solr c:backuprestore s:shard1 r:core_node2 x:backuprestore_shard1_replica_n1] o.a.s.s.IndexSchema [backuprestore_shard1_replica_n1] Schema name=minimal
   [junit4]   2> 3306456 INFO  (qtp1531139025-8207) [n:127.0.0.1:47294_solr c:backuprestore s:shard1 r:core_node2 x:backuprestore_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 3306456 INFO  (qtp1531139025-8207) [n:127.0.0.1:47294_solr c:backuprestore s:shard1 r:core_node2 x:backuprestore_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'backuprestore_shard1_replica_n1' using configuration from collection backuprestore, trusted=true
   [junit4]   2> 3306497 INFO  (qtp1531139025-8211) [n:127.0.0.1:47294_solr c:backuprestore s:shard2 r:core_node15 x:backuprestore_shard2_replica_t13] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_47294.solr.core.backuprestore.shard2.replica_t13' (registry 'solr.core.backuprestore.shard2.replica_t13') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@47e6ecf2
   [junit4]   2> 3306497 INFO  (qtp1531139025-8211) [n:127.0.0.1:47294_solr c:backuprestore s:shard2 r:core_node15 x:backuprestore_shard2_replica_t13] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 3306497 INFO  (qtp1531139025-8211) [n:127.0.0.1:47294_solr c:backuprestore s:shard2 r:core_node15 x:backuprestore_shard2_replica_t13] o.a.s.c.SolrCore [[backuprestore_shard2_replica_t13] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_317B471B1312A108-001/tempDir-001/node2/backuprestore_shard2_replica_t13], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_317B471B1312A108-001/tempDir-001/node2/./backuprestore_shard2_replica_t13/data/]
   [junit4]   2> 3306499 INFO  (qtp229150333-8218) [n:127.0.0.1:57269_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_p7] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 3306500 INFO  (qtp229150333-8218) [n:127.0.0.1:57269_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_p7] o.a.s.c.CoreContainer Creating SolrCore 'backuprestore_shard1_replica_p7' using configuration from collection backuprestore, trusted=true
   [junit4]   2> 3306500 INFO  (qtp229150333-8218) [n:127.0.0.1:57269_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_p7] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_57269.solr.core.backuprestore.shard1.replica_p7' (registry 'solr.core.backuprestore.shard1.replica_p7') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@47e6ecf2
   [junit4]   2> 3306500 INFO  (qtp229150333-8218) [n:127.0.0.1:57269_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_p7] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 3306500 INFO  (qtp229150333-8218) [n:127.0.0.1:57269_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_p7] o.a.s.c.SolrCore [[backuprestore_shard1_replica_p7] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_317B471B1312A108-001/tempDir-001/node1/backuprestore_shard1_replica_p7], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_317B471B1312A108-001/tempDir-001/node1/./backuprestore_shard1_replica_p7/data/]
   [junit4]   2> 3306502 INFO  (qtp229150333-8220) [n:127.0.0.1:57269_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_n11] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 3306502 INFO  (qtp229150333-8220) [n:127.0.0.1:57269_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_n11] o.a.s.c.CoreContainer Creating SolrCore 'backuprestore_shard2_replica_n11' using configuration from collection backuprestore, trusted=true
   [junit4]   2> 3306518 INFO  (qtp229150333-8220) [n:127.0.0.1:57269_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_n11] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_57269.solr.core.backuprestore.shard2.replica_n11' (registry 'solr.core.backuprestore.shard2.replica_n11') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@47e6ecf2
   [junit4]   2> 3306518 INFO  (qtp229150333-8220) [n:127.0.0.1:57269_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_n11] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 3306518 INFO  (qtp229150333-8220) [n:127.0.0.1:57269_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_n11] o.a.s.c.SolrCore [[backuprestore_shard2_replica_n11] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_317B471B1312A108-001/tempDir-001/node1/backuprestore_shard2_replica_n11], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_317B471B1312A108-001/tempDir-001/node1/./backuprestore_shard2_replica_n11/data/]
   [junit4]   2> 3306725 INFO  (qtp1531139025-8207) [n:127.0.0.1:47294_solr c:backuprestore s:shard1 r:core_node2 x:backuprestore_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_47294.solr.core.backuprestore.shard1.replica_n1' (registry 'solr.core.backuprestore.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@47e6ecf2
   [junit4]   2> 3306725 INFO  (qtp1531139025-8207) [n:127.0.0.1:47294_solr c:backuprestore s:shard1 r:core_node2 x:backuprestore_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 3306725 INFO  (qtp1531139025-8207) [n:127.0.0.1:47294_solr c:backuprestore s:shard1 r:core_node2 x:backuprestore_shard1_replica_n1] o.a.s.c.SolrCore [[backuprestore_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_317B471B1312A108-001/tempDir-001/node2/backuprestore_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_317B471B1312A108-001/tempDir-001/node2/./backuprestore_shard1_replica_n1/data/]
   [junit4]   2> 3307948 INFO  (qtp229150333-8217) [n:127.0.0.1:57269_solr c:backuprestore s:shard1 r:core_node4 x:backuprestore_shard1_replica_n3] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 3307948 INFO  (qtp229150333-8217) [n:127.0.0.1:57269_solr c:backuprestore s:shard1 r:core_node4 x:backuprestore_shard1_replica_n3] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 3307949 INFO  (qtp229150333-8217) [n:127.0.0.1:57269_solr c:backuprestore s:shard1 r:core_node4 x:backuprestore_shard1_replica_n3] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 3307949 INFO  (qtp229150333-8217) [n:127.0.0.1:57269_solr c:backuprestore s:shard1 r:core_node4 x:backuprestore_shard1_replica_n3] o.a.s.u.CommitTracker Soft AutoCommit

[...truncated too long message...]

 INFO  (coreCloseExecutor-884-thread-1) [n:127.0.0.1:46367_solr c:foo s:shard1 r:core_node4 x:foo_shard1_replica_n3] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@2bd59d31: rootName = solr_46367, domain = solr.core.foo.shard1.replica_n3, service url = null, agent id = null] for registry solr.core.foo.shard1.replica_n3 / com.codahale.metrics.MetricRegistry@282db3d0
   [junit4]   2> 695386 INFO  (coreCloseExecutor-885-thread-4) [n:127.0.0.1:57824_solr c:multicollection1 s:shard2 r:core_node4 x:multicollection1_shard2_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.multicollection1.shard2.replica_n2, tag=1468663982
   [junit4]   2> 695386 INFO  (coreCloseExecutor-885-thread-4) [n:127.0.0.1:57824_solr c:multicollection1 s:shard2 r:core_node4 x:multicollection1_shard2_replica_n2] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@31e90732: rootName = solr_57824, domain = solr.core.multicollection1.shard2.replica_n2, service url = null, agent id = null] for registry solr.core.multicollection1.shard2.replica_n2 / com.codahale.metrics.MetricRegistry@5c9c280d
   [junit4]   2> 695386 INFO  (coreCloseExecutor-883-thread-4) [n:127.0.0.1:51897_solr c:multicollection1 s:shard1 r:core_node3 x:multicollection1_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.multicollection1.shard1.leader, tag=81560481
   [junit4]   2> 695636 INFO  (coreCloseExecutor-885-thread-5) [n:127.0.0.1:57824_solr c:localShardsTestColl s:shard1 r:core_node7 x:localShardsTestColl_shard1_replica_n4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.localShardsTestColl.shard1.replica_n4, tag=1722637580
   [junit4]   2> 695636 INFO  (coreCloseExecutor-885-thread-5) [n:127.0.0.1:57824_solr c:localShardsTestColl s:shard1 r:core_node7 x:localShardsTestColl_shard1_replica_n4] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@66117c90: rootName = solr_57824, domain = solr.core.localShardsTestColl.shard1.replica_n4, service url = null, agent id = null] for registry solr.core.localShardsTestColl.shard1.replica_n4 / com.codahale.metrics.MetricRegistry@3ba7c26c
   [junit4]   2> 695818 INFO  (coreCloseExecutor-884-thread-1) [n:127.0.0.1:46367_solr c:foo s:shard1 r:core_node4 x:foo_shard1_replica_n3] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.foo.shard1.leader, tag=1727311193
   [junit4]   2> 695845 INFO  (coreCloseExecutor-884-thread-3) [n:127.0.0.1:46367_solr c:multicollection2 s:shard2 r:core_node4 x:multicollection2_shard2_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.multicollection2.shard2.leader, tag=82390428
   [junit4]   2> 695850 INFO  (coreCloseExecutor-885-thread-6) [n:127.0.0.1:57824_solr c:localShardsTestColl s:shard3 r:core_node18 x:localShardsTestColl_shard3_replica_n16] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.localShardsTestColl.shard3.replica_n16, tag=150773247
   [junit4]   2> 695850 INFO  (coreCloseExecutor-885-thread-6) [n:127.0.0.1:57824_solr c:localShardsTestColl s:shard3 r:core_node18 x:localShardsTestColl_shard3_replica_n16] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@1a4d9292: rootName = solr_57824, domain = solr.core.localShardsTestColl.shard3.replica_n16, service url = null, agent id = null] for registry solr.core.localShardsTestColl.shard3.replica_n16 / com.codahale.metrics.MetricRegistry@aa96c9a
   [junit4]   2> 695871 INFO  (coreCloseExecutor-885-thread-5) [n:127.0.0.1:57824_solr c:localShardsTestColl s:shard1 r:core_node7 x:localShardsTestColl_shard1_replica_n4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.localShardsTestColl.shard1.leader, tag=1722637580
   [junit4]   2> 695878 INFO  (coreCloseExecutor-885-thread-4) [n:127.0.0.1:57824_solr c:multicollection1 s:shard2 r:core_node4 x:multicollection1_shard2_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.multicollection1.shard2.leader, tag=1468663982
   [junit4]   2> 695954 INFO  (coreCloseExecutor-885-thread-6) [n:127.0.0.1:57824_solr c:localShardsTestColl s:shard3 r:core_node18 x:localShardsTestColl_shard3_replica_n16] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.localShardsTestColl.shard3.leader, tag=150773247
   [junit4]   2> 696970 WARN  (zkCallback-119-thread-2-processing-n:127.0.0.1:51897_solr) [n:127.0.0.1:51897_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 696971 INFO  (jetty-closer-104-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@2ee099e6{/solr,null,UNAVAILABLE}
   [junit4]   2> 696987 INFO  (zkCallback-120-thread-5-processing-n:127.0.0.1:57824_solr) [n:127.0.0.1:57824_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (1)
   [junit4]   2> 696987 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f12b48d610007, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 697042 INFO  (jetty-closer-104-thread-3) [    ] o.a.s.c.Overseer Overseer (id=98818283500797961-127.0.0.1:57824_solr-n_0000000000) closing
   [junit4]   2> 697042 INFO  (OverseerStateUpdate-98818283500797961-127.0.0.1:57824_solr-n_0000000000) [n:127.0.0.1:57824_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:57824_solr
   [junit4]   2> 697106 WARN  (OverseerAutoScalingTriggerThread-98818283500797961-127.0.0.1:57824_solr-n_0000000000) [n:127.0.0.1:57824_solr    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 698486 WARN  (zkCallback-118-thread-3-processing-n:127.0.0.1:46367_solr) [n:127.0.0.1:46367_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 698486 INFO  (jetty-closer-104-thread-2) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@cd6badc{/solr,null,UNAVAILABLE}
   [junit4]   2> 698634 WARN  (zkCallback-120-thread-5-processing-n:127.0.0.1:57824_solr) [n:127.0.0.1:57824_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 698634 INFO  (jetty-closer-104-thread-3) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@3fba707c{/solr,null,UNAVAILABLE}
   [junit4]   2> 698650 ERROR (SUITE-CloudSolrClientTest-seed#[F4F708FE41B56716]-worker) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 698651 INFO  (SUITE-CloudSolrClientTest-seed#[F4F708FE41B56716]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:33819 33819
   [junit4]   2> 698802 INFO  (Thread-410) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:33819 33819
   [junit4]   2> 698803 WARN  (Thread-410) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	8	/solr/aliases.json
   [junit4]   2> 	4	/solr/configs/conf
   [junit4]   2> 	3	/solr/security.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	30	/solr/collections/foo/state.json
   [junit4]   2> 	30	/solr/collections/localShardsTestColl/state.json
   [junit4]   2> 	17	/solr/collections/collection1/state.json
   [junit4]   2> 	16	/solr/collections/2nd_collection/state.json
   [junit4]   2> 	14	/solr/collections/nemesis/state.json
   [junit4]   2> 	14	/solr/collections/multicollection2/state.json
   [junit4]   2> 	14	/solr/collections/multicollection1/state.json
   [junit4]   2> 	8	/solr/clusterprops.json
   [junit4]   2> 	8	/solr/clusterstate.json
   [junit4]   2> 	6	/solr/collections/overwrite/state.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	8	/solr/live_nodes
   [junit4]   2> 	8	/solr/collections
   [junit4]   2> 
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-solrj/test/J2/temp/solr.client.solrj.impl.CloudSolrClientTest_F4F708FE41B56716-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70), sim=RandomSimilarity(queryNorm=false): {}, locale=ar-QA, timezone=Europe/Warsaw
   [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=172377944,total=410517504
   [junit4]   2> NOTE: All tests run in this JVM: [SchemaTest, TestJavaBinCodec, LessThanEvaluatorTest, LargeVolumeJettyTest, EmpiricalDistributionEvaluatorTest, FieldValueEvaluatorTest, TestCoreAdmin, TestV1toV2ApiMapper, NotEvaluatorTest, SquareRootEvaluatorTest, TestToleratedUpdateError, ArcSineEvaluatorTest, GreaterThanEvaluatorTest, SolrExampleBinaryTest, TestHash, SolrExampleJettyTest, ModifiableSolrParamsTest, SolrParamTest, JettyWebappTest, CloudSolrClientCacheTest, StreamExpressionParserTest, TestPolicy, AscEvaluatorTest, CloudSolrClientMultiConstructorTest, CloudSolrClientTest]
   [junit4] Completed [101/145 (1!)] on J2 in 189.35s, 15 tests, 1 error <<< FAILURES!

[...truncated 46084 lines...]

[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1407 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1407/

6 tests failed.
FAILED:  org.apache.lucene.index.TestBagOfPositions.test

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
	at __randomizedtesting.SeedInfo.seed([F318A0E5FC5C51D]:0)


FAILED:  junit.framework.TestSuite.org.apache.lucene.index.TestBagOfPositions

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
	at __randomizedtesting.SeedInfo.seed([F318A0E5FC5C51D]:0)


FAILED:  org.apache.solr.cloud.CollectionsAPISolrJTest.testCreateAndDeleteShard

Error Message:
Error from server at https://127.0.0.1:44830/solr: KeeperErrorCode = NoNode for /overseer/collection-queue-work/qnr-0000000028

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:44830/solr: KeeperErrorCode = NoNode for /overseer/collection-queue-work/qnr-0000000028
	at __randomizedtesting.SeedInfo.seed([C28AB6B61C219EA8:7D402AA9AD45455]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:626)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1086)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:867)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:800)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
	at org.apache.solr.cloud.CollectionsAPISolrJTest.testCreateAndDeleteShard(CollectionsAPISolrJTest.java:157)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.CustomCollectionTest.testRouteFieldForHashRouter

Error Message:
Error from server at https://127.0.0.1:53559/solr: KeeperErrorCode = NoNode for /overseer/collection-queue-work/qnr-0000000002

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:53559/solr: KeeperErrorCode = NoNode for /overseer/collection-queue-work/qnr-0000000002
	at __randomizedtesting.SeedInfo.seed([C28AB6B61C219EA8:6ABC286B834075F2]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:626)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1086)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:867)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:800)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
	at org.apache.solr.cloud.MiniSolrCloudCluster.deleteAllCollections(MiniSolrCloudCluster.java:444)
	at org.apache.solr.cloud.CustomCollectionTest.ensureClusterEmpty(CustomCollectionTest.java:53)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:968)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test

Error Message:
Captured an uncaught exception in thread: Thread[id=2937, name=Thread-551, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=2937, name=Thread-551, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:60801/jt/d/collection2_shard1_replica_n1: Cannot talk to ZooKeeper - Updates are disabled.
	at __randomizedtesting.SeedInfo.seed([C28AB6B61C219EA8]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:626)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:551)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:998)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:867)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:800)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest$1IndexThread.run(FullSolrCloudDistribCmdsTest.java:633)


FAILED:  org.apache.solr.cloud.hdfs.HdfsUnloadDistributedZkTest.test

Error Message:
Captured an uncaught exception in thread: Thread[id=57390, name=testExecutor-10985-thread-12, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=57390, name=testExecutor-10985-thread-12, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest]
Caused by: java.lang.RuntimeException: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:59451/oyf/qa
	at __randomizedtesting.SeedInfo.seed([C28AB6B61C219EA8]:0)
	at org.apache.solr.cloud.BasicDistributedZkTest.lambda$createCollectionInOneInstance$0(BasicDistributedZkTest.java:595)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:59451/oyf/qa
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:637)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
	at org.apache.solr.cloud.BasicDistributedZkTest.lambda$createCollectionInOneInstance$0(BasicDistributedZkTest.java:593)
	... 4 more
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:171)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:525)
	... 9 more




Build Log:
[...truncated 1739 lines...]
   [junit4] Suite: org.apache.lucene.index.TestBagOfPositions
   [junit4]   2> 10 24, 0029 4:37:17 午前 com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
   [junit4]   2> WARNING: Suite execution timed out: org.apache.lucene.index.TestBagOfPositions
   [junit4]   2>    1) Thread[id=9, name=JUnit4-serializer-daemon, state=TIMED_WAITING, group=main]
   [junit4]   2>         at java.lang.Thread.sleep(Native Method)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:50)
   [junit4]   2>    2) Thread[id=674, name=Thread-604, state=RUNNABLE, group=TGRP-TestBagOfPositions]
   [junit4]   2>         at java.lang.Thread.yield(Native Method)
   [junit4]   2>         at org.apache.lucene.store.MockIndexOutputWrapper.writeBytes(MockIndexOutputWrapper.java:138)
   [junit4]   2>         at org.apache.lucene.store.DataOutput.writeBytes(DataOutput.java:52)
   [junit4]   2>         at org.apache.lucene.util.packed.PackedWriter.flush(PackedWriter.java:86)
   [junit4]   2>         at org.apache.lucene.util.packed.PackedWriter.finish(PackedWriter.java:79)
   [junit4]   2>         at org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.flushFlags(CompressingTermVectorsWriter.java:457)
   [junit4]   2>         at org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.flush(CompressingTermVectorsWriter.java:343)
   [junit4]   2>         at org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.finishDocument(CompressingTermVectorsWriter.java:276)
   [junit4]   2>         at org.apache.lucene.codecs.asserting.AssertingTermVectorsFormat$AssertingTermVectorsWriter.finishDocument(AssertingTermVectorsFormat.java:142)
   [junit4]   2>         at org.apache.lucene.codecs.TermVectorsWriter.addAllDocVectors(TermVectorsWriter.java:322)
   [junit4]   2>         at org.apache.lucene.codecs.TermVectorsWriter.merge(TermVectorsWriter.java:225)
   [junit4]   2>         at org.apache.lucene.index.SegmentMerger.mergeVectors(SegmentMerger.java:224)
   [junit4]   2>         at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:158)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4390)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4030)
   [junit4]   2>         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2233)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5045)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1718)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1460)
   [junit4]   2>         at org.apache.lucene.index.RandomIndexWriter.addDocument(RandomIndexWriter.java:171)
   [junit4]   2>         at org.apache.lucene.index.TestBagOfPositions$1.run(TestBagOfPositions.java:119)
   [junit4]   2>    3) Thread[id=672, name=SUITE-TestBagOfPositions-seed#[F318A0E5FC5C51D], state=RUNNABLE, group=TGRP-TestBagOfPositions]
   [junit4]   2>         at java.lang.Thread.getStackTrace(Thread.java:1559)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:696)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:693)
   [junit4]   2>         at java.security.AccessController.doPrivileged(Native Method)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getStackTrace(ThreadLeakControl.java:693)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getThreadsWithTraces(ThreadLeakControl.java:709)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.formatThreadStacksFull(ThreadLeakControl.java:689)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.access$1000(ThreadLeakControl.java:65)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$2.evaluate(ThreadLeakControl.java:415)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:705)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.access$200(RandomizedRunner.java:139)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$2.run(RandomizedRunner.java:626)
   [junit4]   2>    4) Thread[id=677, name=Thread-607, state=BLOCKED, group=TGRP-TestBagOfPositions]
   [junit4]   2>         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:36)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2233)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5045)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1718)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1460)
   [junit4]   2>         at org.apache.lucene.index.RandomIndexWriter.addDocument(RandomIndexWriter.java:171)
   [junit4]   2>         at org.apache.lucene.index.TestBagOfPositions$1.run(TestBagOfPositions.java:119)
   [junit4]   2>    5) Thread[id=678, name=Thread-608, state=BLOCKED, group=TGRP-TestBagOfPositions]
   [junit4]   2>         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:36)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2233)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5045)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1718)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1460)
   [junit4]   2>         at org.apache.lucene.index.RandomIndexWriter.addDocument(RandomIndexWriter.java:171)
   [junit4]   2>         at org.apache.lucene.index.TestBagOfPositions$1.run(TestBagOfPositions.java:119)
   [junit4]   2>    6) Thread[id=1, name=main, state=WAITING, group=main]
   [junit4]   2>         at java.lang.Object.wait(Native Method)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1252)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1326)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:636)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:493)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:251)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
   [junit4]   2>    7) Thread[id=675, name=Thread-605, state=BLOCKED, group=TGRP-TestBagOfPositions]
   [junit4]   2>         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:36)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2233)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5045)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1718)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1460)
   [junit4]   2>         at org.apache.lucene.index.RandomIndexWriter.addDocument(RandomIndexWriter.java:171)
   [junit4]   2>         at org.apache.lucene.index.TestBagOfPositions$1.run(TestBagOfPositions.java:119)
   [junit4]   2>    8) Thread[id=676, name=Thread-606, state=BLOCKED, group=TGRP-TestBagOfPositions]
   [junit4]   2>         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:36)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2233)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5045)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1531)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1504)
   [junit4]   2>         at org.apache.lucene.index.RandomIndexWriter.addDocument(RandomIndexWriter.java:141)
   [junit4]   2>         at org.apache.lucene.index.TestBagOfPositions$1.run(TestBagOfPositions.java:119)
   [junit4]   2>    9) Thread[id=673, name=TEST-TestBagOfPositions.test-seed#[F318A0E5FC5C51D], state=WAITING, group=TGRP-TestBagOfPositions]
   [junit4]   2>         at java.lang.Object.wait(Native Method)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1252)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1326)
   [junit4]   2>         at org.apache.lucene.index.TestBagOfPositions.test(TestBagOfPositions.java:130)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestBagOfPositions -Dtests.method=test -Dtests.seed=F318A0E5FC5C51D -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=ja-JP-u-ca-japanese-x-lvariant-JP -Dtests.timezone=America/Danmarkshavn -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   7201s J0 | TestBagOfPositions.test <<<
   [junit4]    > Throwable #1: java.lang.Exception: Test abandoned because suite timeout was reached.
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([F318A0E5FC5C51D]:0)
   [junit4]   2> 10 24, 0029 4:37:18 午前 com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 6 leaked thread(s).
   [junit4]   2> 10 24, 0029 4:37:38 午前 com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 6 threads leaked from SUITE scope at org.apache.lucene.index.TestBagOfPositions: 
   [junit4]   2>    1) Thread[id=674, name=Thread-604, state=BLOCKED, group=TGRP-TestBagOfPositions]
   [junit4]   2>         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:36)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2233)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5045)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1531)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1504)
   [junit4]   2>         at org.apache.lucene.index.RandomIndexWriter.addDocument(RandomIndexWriter.java:141)
   [junit4]   2>         at org.apache.lucene.index.TestBagOfPositions$1.run(TestBagOfPositions.java:119)
   [junit4]   2>    2) Thread[id=677, name=Thread-607, state=BLOCKED, group=TGRP-TestBagOfPositions]
   [junit4]   2>         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:36)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2233)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5045)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1718)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1460)
   [junit4]   2>         at org.apache.lucene.index.RandomIndexWriter.addDocument(RandomIndexWriter.java:171)
   [junit4]   2>         at org.apache.lucene.index.TestBagOfPositions$1.run(TestBagOfPositions.java:119)
   [junit4]   2>    3) Thread[id=678, name=Thread-608, state=RUNNABLE, group=TGRP-TestBagOfPositions]
   [junit4]   2>         at org.apache.lucene.codecs.compressing.CompressingTermVectorsReader.get(CompressingTermVectorsReader.java:391)
   [junit4]   2>         at org.apache.lucene.codecs.asserting.AssertingTermVectorsFormat$AssertingTermVectorsReader.get(AssertingTermVectorsFormat.java:71)
   [junit4]   2>         at org.apache.lucene.codecs.TermVectorsWriter.merge(TermVectorsWriter.java:223)
   [junit4]   2>         at org.apache.lucene.index.SegmentMerger.mergeVectors(SegmentMerger.java:224)
   [junit4]   2>         at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:158)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4390)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4030)
   [junit4]   2>         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2233)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5045)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1718)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1460)
   [junit4]   2>         at org.apache.lucene.index.RandomIndexWriter.addDocument(RandomIndexWriter.java:171)
   [junit4]   2>         at org.apache.lucene.index.TestBagOfPositions$1.run(TestBagOfPositions.java:119)
   [junit4]   2>    4) Thread[id=675, name=Thread-605, state=BLOCKED, group=TGRP-TestBagOfPositions]
   [junit4]   2>         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:36)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2233)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5045)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1718)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1460)
   [junit4]   2>         at org.apache.lucene.index.RandomIndexWriter.addDocument(RandomIndexWriter.java:171)
   [junit4]   2>         at org.apache.lucene.index.TestBagOfPositions$1.run(TestBagOfPositions.java:119)
   [junit4]   2>    5) Thread[id=676, name=Thread-606, state=BLOCKED, group=TGRP-TestBagOfPositions]
   [junit4]   2>         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:36)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2233)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5045)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1718)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1460)
   [junit4]   2>         at org.apache.lucene.index.RandomIndexWriter.addDocument(RandomIndexWriter.java:171)
   [junit4]   2>         at org.apache.lucene.index.TestBagOfPositions$1.run(TestBagOfPositions.java:119)
   [junit4]   2>    6) Thread[id=673, name=TEST-TestBagOfPositions.test-seed#[F318A0E5FC5C51D], state=WAITING, group=TGRP-TestBagOfPositions]
   [junit4]   2>         at java.lang.Object.wait(Native Method)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1252)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1326)
   [junit4]   2>         at org.apache.lucene.index.TestBagOfPositions.test(TestBagOfPositions.java:130)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 10 24, 0029 4:37:38 午前 com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=674, name=Thread-604, state=BLOCKED, group=TGRP-TestBagOfPositions]
   [junit4]   2>    2) Thread[id=677, name=Thread-607, state=BLOCKED, group=TGRP-TestBagOfPositions]
   [junit4]   2>    3) Thread[id=678, name=Thread-608, state=RUNNABLE, group=TGRP-TestBagOfPositions]
   [junit4]   2>    4) Thread[id=675, name=Thread-605, state=BLOCKED, group=TGRP-TestBagOfPositions]
   [junit4]   2>    5) Thread[id=676, name=Thread-606, state=BLOCKED, group=TGRP-TestBagOfPositions]
   [junit4]   2>    6) Thread[id=673, name=TEST-TestBagOfPositions.test-seed#[F318A0E5FC5C51D], state=WAITING, group=TGRP-TestBagOfPositions]
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.index.TestBagOfPositions_F318A0E5FC5C51D-001
   [junit4]   2> Oct 24, 2017 4:37:38 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: All leaked threads terminated.
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {field=PostingsFormat(name=MockRandom)}, docValues:{}, maxPointsInLeafNode=1087, maxMBSortInHeap=5.632682684864807, sim=RandomSimilarity(queryNorm=false): {}, locale=ja-JP-u-ca-japanese-x-lvariant-JP, timezone=America/Danmarkshavn
   [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=131093040,total=534773760
   [junit4]   2> NOTE: All tests run in this JVM: [TestBinaryTerms, TestFloatRangeFieldQueries, TestIndexWriterForceMerge, TestBooleanScorer, TestIndexedDISI, TestCrashCausesCorruptIndex, TestCodecs, TestBooleanSimilarity, TestIndexWriterExceptions, TestTerm, TestNewestSegment, TestIntRangeFieldQueries, TestTimSorterWorstCase, TestNoDeletionPolicy, TestLiveFieldValues, TestUpgradeIndexMergePolicy, TestMultiThreadTermVectors, TestFieldMaskingSpanQuery, TestIntBlockPool, TestSnapshotDeletionPolicy, TestSearchAfter, TestSloppyPhraseQuery, TestMathUtil, TestFixedLengthBytesRefArray, TestPersistentSnapshotDeletionPolicy, TestDoubleRangeFieldQueries, TestTermVectorsWriter, TestUTF32ToUTF8, TestDeterminizeLexicon, TestBagOfPositions]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestBagOfPositions -Dtests.seed=F318A0E5FC5C51D -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=ja-JP-u-ca-japanese-x-lvariant-JP -Dtests.timezone=America/Danmarkshavn -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J0 | TestBagOfPositions (suite) <<<
   [junit4]    > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([F318A0E5FC5C51D]:0)
   [junit4] Completed [305/455 (1!)] on J0 in 7222.96s, 1 test, 2 errors <<< FAILURES!

[...truncated 11398 lines...]
   [junit4] Suite: org.apache.solr.cloud.FullSolrCloudDistribCmdsTest
   [junit4]   2> 1617939 INFO  (SUITE-FullSolrCloudDistribCmdsTest-seed#[C28AB6B61C219EA8]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_C28AB6B61C219EA8-001/init-core-data-001
   [junit4]   2> 1617939 INFO  (SUITE-FullSolrCloudDistribCmdsTest-seed#[C28AB6B61C219EA8]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 1618049 INFO  (SUITE-FullSolrCloudDistribCmdsTest-seed#[C28AB6B61C219EA8]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
   [junit4]   2> 1618049 INFO  (SUITE-FullSolrCloudDistribCmdsTest-seed#[C28AB6B61C219EA8]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /jt/d
   [junit4]   2> 1618060 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1618068 INFO  (Thread-469) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1618068 INFO  (Thread-469) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1618169 ERROR (Thread-469) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 1618184 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.c.ZkTestServer start zk server on port:55421
   [junit4]   2> 1618401 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1618403 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 1618404 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1618406 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1618407 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1618408 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1618409 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1618410 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1618411 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1618411 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1618412 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1618460 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
   [junit4]   2> 1621112 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1621140 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@71d30da4{/jt/d,null,AVAILABLE}
   [junit4]   2> 1621152 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@29659ec8{HTTP/1.1,[http/1.1]}{127.0.0.1:51073}
   [junit4]   2> 1621152 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.e.j.s.Server Started @1640192ms
   [junit4]   2> 1621152 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_C28AB6B61C219EA8-001/tempDir-001/control/data, hostContext=/jt/d, hostPort=51073, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_C28AB6B61C219EA8-001/control-001/cores}
   [junit4]   2> 1621152 ERROR (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1621207 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 1621207 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1621207 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 1621207 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-24T12:35:25.629Z
   [junit4]   2> 1621237 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 1621237 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_C28AB6B61C219EA8-001/control-001/solr.xml
   [junit4]   2> 1621260 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 1621260 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 1621391 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@5acc2ec, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 1621396 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:55421/solr
   [junit4]   2> 1621948 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [n:127.0.0.1:51073_jt%2Fd    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1621948 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [n:127.0.0.1:51073_jt%2Fd    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:51073_jt%2Fd
   [junit4]   2> 1621952 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [n:127.0.0.1:51073_jt%2Fd    ] o.a.s.c.Overseer Overseer (id=98883896770953220-127.0.0.1:51073_jt%2Fd-n_0000000000) starting
   [junit4]   2> 1622153 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [n:127.0.0.1:51073_jt%2Fd    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51073_jt%2Fd
   [junit4]   2> 1622168 INFO  (zkCallback-421-thread-1-processing-n:127.0.0.1:51073_jt%2Fd) [n:127.0.0.1:51073_jt%2Fd    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1622983 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [n:127.0.0.1:51073_jt%2Fd    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5acc2ec
   [junit4]   2> 1623035 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [n:127.0.0.1:51073_jt%2Fd    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5acc2ec
   [junit4]   2> 1623035 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [n:127.0.0.1:51073_jt%2Fd    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5acc2ec
   [junit4]   2> 1623060 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [n:127.0.0.1:51073_jt%2Fd    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_C28AB6B61C219EA8-001/control-001/cores
   [junit4]   2> 1623242 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1623243 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:55421/solr ready
   [junit4]   2> 1623531 INFO  (qtp911616473-2464) [n:127.0.0.1:51073_jt%2Fd    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:51073_jt%252Fd&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1623652 INFO  (OverseerThreadFactory-1290-thread-1-processing-n:127.0.0.1:51073_jt%2Fd) [n:127.0.0.1:51073_jt%2Fd    ] o.a.s.c.CreateCollectionCmd Create collection control_collection
   [junit4]   2> 1623893 INFO  (qtp911616473-2462) [n:127.0.0.1:51073_jt%2Fd    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1623894 INFO  (qtp911616473-2462) [n:127.0.0.1:51073_jt%2Fd    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 1624028 INFO  (zkCallback-421-thread-1-processing-n:127.0.0.1:51073_jt%2Fd) [n:127.0.0.1:51073_jt%2Fd    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1625147 INFO  (qtp911616473-2462) [n:127.0.0.1:51073_jt%2Fd c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 1625168 INFO  (qtp911616473-2462) [n:127.0.0.1:51073_jt%2Fd c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
   [junit4]   2> 1626263 INFO  (qtp911616473-2462) [n:127.0.0.1:51073_jt%2Fd c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 1626441 INFO  (qtp911616473-2462) [n:127.0.0.1:51073_jt%2Fd c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
   [junit4]   2> 1626442 INFO  (qtp911616473-2462) [n:127.0.0.1:51073_jt%2Fd c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5acc2ec
   [junit4]   2> 1626442 INFO  (qtp911616473-2462) [n:127.0.0.1:51073_jt%2Fd c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1626442 INFO  (qtp911616473-2462) [n:127.0.0.1:51073_jt%2Fd c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_C28AB6B61C219EA8-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_C28AB6B61C219EA8-001/control-001/cores/control_collection_shard1_replica_n1/data/]
   [junit4]   2> 1626504 INFO  (qtp911616473-2462) [n:127.0.0.1:51073_jt%2Fd c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@1c416f70
   [junit4]   2> 1626597 WARN  (qtp911616473-2462) [n:127.0.0.1:51073_jt%2Fd c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 1627160 INFO  (qtp911616473-2462) [n:127.0.0.1:51073_jt%2Fd c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1627160 INFO  (qtp911616473-2462) [n:127.0.0.1:51073_jt%2Fd c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1627161 INFO  (qtp911616473-2462) [n:127.0.0.1:51073_jt%2Fd c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1627161 INFO  (qtp911616473-2462) [n:127.0.0.1:51073_jt%2Fd c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1627200 INFO  (qtp911616473-2462) [n:127.0.0.1:51073_jt%2Fd c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=29, maxMergeAtOnceExplicit=33, maxMergedSegmentMB=96.2294921875, floorSegmentMB=1.0966796875, forceMergeDeletesPctAllowed=17.145537848379604, segmentsPerTier=49.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6417866155273849
   [junit4]   2> 1627200 INFO  (qtp911616473-2462) [n:127.0.0.1:51073_jt%2Fd c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@f068131[control_collection_shard1_replica_n1] main]
   [junit4]   2> 1627201 INFO  (qtp911616473-2462) [n:127.0.0.1:51073_jt%2Fd c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1627202 INFO  (qtp911616473-2462) [n:127.0.0.1:51073_jt%2Fd c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1627202 INFO  (qtp911616473-2462) [n:127.0.0.1:51073_jt%2Fd c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 1627203 INFO  (searcherExecutor-1293-thread-1-processing-n:127.0.0.1:51073_jt%2Fd x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:51073_jt%2Fd c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@f068131[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1627204 INFO  (qtp911616473-2462) [n:127.0.0.1:51073_jt%2Fd c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1582142357898264576
   [junit4]   2> 1627239 INFO  (qtp911616473-2462) [n:127.0.0.1:51073_jt%2Fd c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1627239 INFO  (qtp911616473-2462) [n:127.0.0.1:51073_jt%2Fd c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1627239 INFO  (qtp911616473-2462) [n:127.0.0.1:51073_jt%2Fd c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:51073/jt/d/control_collection_shard1_replica_n1/
   [junit4]   2> 1627239 INFO  (qtp911616473-2462) [n:127.0.0.1:51073_jt%2Fd c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 1627239 INFO  (qtp911616473-2462) [n:127.0.0.1:51073_jt%2Fd c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:51073/jt/d/control_collection_shard1_replica_n1/ has no replicas
   [junit4]   2> 1627239 INFO  (qtp911616473-2462) [n:127.0.0.1:51073_jt%2Fd c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 1627258 INFO  (zkCallback-421-thread-2-processing-n:127.0.0.1:51073_jt%2Fd) [n:127.0.0.1:51073_jt%2Fd    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1627259 INFO  (qtp911616473-2462) [n:127.0.0.1:51073_jt%2Fd c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:51073/jt/d/control_collection_shard1_replica_n1/ shard1
   [junit4]   2> 1627259 INFO  (qtp911616473-2462) [n:127.0.0.1:51073_jt%2Fd c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1627268 INFO  (qtp911616473-2462) [n:127.0.0.1:51073_jt%2Fd c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=3375
   [junit4]   2> 1627492 INFO  (zkCallback-421-thread-1-processing-n:127.0.0.1:51073_jt%2Fd) [n:127.0.0.1:51073_jt%2Fd    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1627734 INFO  (qtp911616473-2464) [n:127.0.0.1:51073_jt%2Fd    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 1627734 INFO  (qtp911616473-2464) [n:127.0.0.1:51073_jt%2Fd    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:51073_jt%252Fd&wt=javabin&version=2} status=0 QTime=4203
   [junit4]   2> 1627741 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f4e61550a0005, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1628245 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1628246 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:55421/solr ready
   [junit4]   2> 1628246 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1628247 INFO  (qtp911616473-2468) [n:127.0.0.1:51073_jt%2Fd    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=3&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1628253 INFO  (OverseerThreadFactory-1290-thread-2-processing-n:127.0.0.1:51073_jt%2Fd) [n:127.0.0.1:51073_jt%2Fd    ] o.a.s.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 1628286 WARN  (OverseerThreadFactory-1290-thread-2-processing-n:127.0.0.1:51073_jt%2Fd) [n:127.0.0.1:51073_jt%2Fd    ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
   [junit4]   2> 1628414 INFO  (qtp911616473-2468) [n:127.0.0.1:51073_jt%2Fd    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 1628414 INFO  (qtp911616473-2468) [n:127.0.0.1:51073_jt%2Fd    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=3&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=167
   [junit4]   2> 1628420 INFO  (OverseerCollectionConfigSetProcessor-98883896770953220-127.0.0.1:51073_jt%2Fd-n_0000000000) [n:127.0.0.1:51073_jt%2Fd    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1629200 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_C28AB6B61C219EA8-001/shard-1-001 of type NRT
   [junit4]   2> 1629200 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1629212 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3e15feea{/jt/d,null,AVAILABLE}
   [junit4]   2> 1629212 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@3f0faf32{HTTP/1.1,[http/1.1]}{127.0.0.1:60531}
   [junit4]   2> 1629212 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.e.j.s.Server Started @1648252ms
   [junit4]   2> 1629212 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_C28AB6B61C219EA8-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/jt/d, hostPort=60531, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_C28AB6B61C219EA8-001/shard-1-001/cores}
   [junit4]   2> 1629212 ERROR (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1629212 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 1629213 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1629213 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 1629213 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-24T12:35:33.635Z
   [junit4]   2> 1629365 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 1629365 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_C28AB6B61C219EA8-001/shard-1-001/solr.xml
   [junit4]   2> 1629380 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 1629381 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 1629382 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@5acc2ec, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 1629448 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:55421/solr
   [junit4]   2> 1629534 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [n:127.0.0.1:60531_jt%2Fd    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1629544 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [n:127.0.0.1:60531_jt%2Fd    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1629565 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [n:127.0.0.1:60531_jt%2Fd    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60531_jt%2Fd
   [junit4]   2> 1629565 INFO  (zkCallback-429-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1629565 INFO  (zkCallback-421-thread-1-processing-n:127.0.0.1:51073_jt%2Fd) [n:127.0.0.1:51073_jt%2Fd    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1629642 INFO  (zkCallback-434-thread-1-processing-n:127.0.0.1:60531_jt%2Fd) [n:127.0.0.1:60531_jt%2Fd    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1630763 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [n:127.0.0.1:60531_jt%2Fd    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5acc2ec
   [junit4]   2> 1630836 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [n:127.0.0.1:60531_jt%2Fd    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5acc2ec
   [junit4]   2> 1630991 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [n:127.0.0.1:60531_jt%2Fd    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5acc2ec
   [junit4]   2> 1630994 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [n:127.0.0.1:60531_jt%2Fd    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_C28AB6B61C219EA8-001/shard-1-001/cores
   [junit4]   2> 1631452 INFO  (qtp911616473-2466) [n:127.0.0.1:51073_jt%2Fd    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:60531_jt%252Fd&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1631461 INFO  (OverseerThreadFactory-1290-thread-3-processing-n:127.0.0.1:51073_jt%2Fd) [n:127.0.0.1:51073_jt%2Fd    ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:60531_jt%2Fd for creating new replica
   [junit4]   2> 1631502 INFO  (qtp718709710-2511) [n:127.0.0.1:60531_jt%2Fd    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n61&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1631502 INFO  (qtp718709710-2511) [n:127.0.0.1:60531_jt%2Fd    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 1632751 INFO  (qtp718709710-2511) [n:127.0.0.1:60531_jt%2Fd c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 1633217 INFO  (qtp718709710-2511) [n:127.0.0.1:60531_jt%2Fd c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.s.IndexSchema [collection1_shard2_replica_n61] Schema name=test
   [junit4]   2> 1633750 INFO  (qtp718709710-2511) [n:127.0.0.1:60531_jt%2Fd c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 1633784 INFO  (qtp718709710-2511) [n:127.0.0.1:60531_jt%2Fd c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_n61' using configuration from collection collection1, trusted=true
   [junit4]   2> 1633785 INFO  (qtp718709710-2511) [n:127.0.0.1:60531_jt%2Fd c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_n61' (registry 'solr.core.collection1.shard2.replica_n61') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5acc2ec
   [junit4]   2> 1633785 INFO  (qtp718709710-2511) [n:127.0.0.1:60531_jt%2Fd c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1633785 INFO  (qtp718709710-2511) [n:127.0.0.1:60531_jt%2Fd c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.c.SolrCore [[collection1_shard2_replica_n61] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_C28AB6B61C219EA8-001/shard-1-001/cores/collection1_shard2_replica_n61], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_C28AB6B61C219EA8-001/shard-1-001/cores/collection1_shard2_replica_n61/data/]
   [junit4]   2> 1633808 INFO  (qtp718709710-2511) [n:127.0.0.1:60531_jt%2Fd c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@363b49f5
   [junit4]   2> 1633834 WARN  (qtp718709710-2511) [n:127.0.0.1:60531_jt%2Fd c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 1634524 INFO  (qtp718709710-2511) [n:127.0.0.1:60531_jt%2Fd c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1634524 INFO  (qtp718709710-2511) [n:127.0.0.1:60531_jt%2Fd c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1634525 INFO  (qtp718709710-2511) [n:127.0.0.1:60531_jt%2Fd c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1634525 INFO  (qtp718709710-2511) [n:127.0.0.1:60531_jt%2Fd c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1634540 INFO  (qtp718709710-2511) [n:127.0.0.1:60531_jt%2Fd c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=29, maxMergeAtOnceExplicit=33, maxMergedSegmentMB=96.2294921875, floorSegmentMB=1.0966796875, forceMergeDeletesPctAllowed=17.145537848379604, segmentsPerTier=49.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6417866155273849
   [junit4]   2> 1634540 INFO  (qtp718709710-2511) [n:127.0.0.1:60531_jt%2Fd c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.s.SolrIndexSearcher Opening [Searcher@16b24eb9[collection1_shard2_replica_n61] main]
   [junit4]   2> 1634541 INFO  (qtp718709710-2511) [n:127.0.0.1:60531_jt%2Fd c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1634542 INFO  (qtp718709710-2511) [n:127.0.0.1:60531_jt%2Fd c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1634542 INFO  (qtp718709710-2511) [n:127.0.0.1:60531_jt%2Fd c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 1634542 INFO  (qtp718709710-2511) [n:127.0.0.1:60531_jt%2Fd c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1582142365592715264
   [junit4]   2> 1634585 INFO  (searcherExecutor-1304-thread-1-processing-n:127.0.0.1:60531_jt%2Fd x:collection1_shard2_replica_n61 s:shard2 c:collection1) [n:127.0.0.1:60531_jt%2Fd c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.c.SolrCore [collection1_shard2_replica_n61] Registered new searcher Searcher@16b24eb9[collection1_shard2_replica_n61] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1634614 INFO  (qtp718709710-2511) [n:127.0.0.1:60531_jt%2Fd c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1634614 INFO  (qtp718709710-2511) [n:127.0.0.1:60531_jt%2Fd c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1634614 INFO  (qtp718709710-2511) [n:127.0.0.1:60531_jt%2Fd c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:60531/jt/d/collection1_shard2_replica_n61/
   [junit4]   2> 1634614 INFO  (qtp718709710-2511) [n:127.0.0.1:60531_jt%2Fd c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 1634614 INFO  (qtp718709710-2511) [n:127.0.0.1:60531_jt%2Fd c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.c.SyncStrategy http://127.0.0.1:60531/jt/d/collection1_shard2_replica_n61/ has no replicas
   [junit4]   2> 1634614 INFO  (qtp718709710-2511) [n:127.0.0.1:60531_jt%2Fd c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 1634616 INFO  (qtp718709710-2511) [n:127.0.0.1:60531_jt%2Fd c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:60531/jt/d/collection1_shard2_replica_n61/ shard2
   [junit4]   2> 1634642 INFO  (qtp718709710-2511) [n:127.0.0.1:60531_jt%2Fd c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1634642 INFO  (qtp718709710-2511) [n:127.0.0.1:60531_jt%2Fd c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n61&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=3140
   [junit4]   2> 1634653 INFO  (qtp911616473-2466) [n:127.0.0.1:51073_jt%2Fd    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:60531_jt%252Fd&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2} status=0 QTime=3201
   [junit4]   2> 1635701 INFO  (OverseerCollectionConfigSetProcessor-98883896770953220-127.0.0.1:51073_jt%2Fd-n_0000000000) [n:127.0.0.1:51073_jt%2Fd    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1636639 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_C28AB6B61C219EA8-001/shard-2-001 of type NRT
   [junit4]   2> 1636639 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1636644 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@54d748d1{/jt/d,null,AVAILABLE}
   [junit4]   2> 1636645 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@5a57c0df{HTTP/1.1,[http/1.1]}{127.0.0.1:60801}
   [junit4]   2> 1636645 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.e.j.s.Server Started @1655684ms
   [junit4]   2> 1636645 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_C28AB6B61C219EA8-001/tempDir-001/jetty2, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/jt/d, hostPort=60801, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_C28AB6B61C219EA8-001/shard-2-001/cores}
   [junit4]   2> 1636645 ERROR (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1636708 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 1636709 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1636709 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 1636709 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-24T12:35:41.131Z
   [junit4]   2> 1636765 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 1636765 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_C28AB6B61C219EA8-001/shard-2-001/solr.xml
   [junit4]   2> 1636809 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 1636810 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 1636811 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@5acc2ec, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 1636833 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:55421/solr
   [junit4]   2> 1637079 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C28AB6B61C219EA8]) [n:127.0.0.1:60801_jt%2Fd    ] o.a.s.c.c.ZkStateReader Updated live nodes fr

[...truncated too long message...]

ut/solr/core/test-lib/apacheds-interceptors-journal-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-normalization-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-operational-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-referral-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-schema-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-subtree-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-trigger-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-jdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-jdbm1-2.0.0-M2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-kerberos-codec-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-ldif-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-mavibot-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-protocol-kerberos-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-protocol-ldap-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-protocol-shared-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-xdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/api-all-1.0.0-M20.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/bcprov-jdk15-1.45.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/byte-buddy-1.6.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/commons-collections-3.2.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/commons-math3-3.6.1.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/ehcache-core-2.4.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/hadoop-common-2.7.4-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/hadoop-hdfs-2.7.4-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/hadoop-minikdc-2.7.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/htrace-core-3.2.0-incubating.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jersey-core-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jersey-server-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jetty-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jetty-sslengine-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jetty-util-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/mina-core-2.0.0-M5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/mockito-core-2.6.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/netty-all-4.0.36.Final.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/objenesis-2.5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/analysis/icu/lucene-analyzers-icu-8.0.0-SNAPSHOT.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/contrib/solr-analysis-extras/classes/java:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/queryparser/classes/test:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/backward-codecs/classes/test:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/contrib/analysis-extras/lib/icu4j-59.1.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-launcher.jar:/x1/jenkins/.ant/lib/ivy-2.3.0.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-log4j.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit4.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jai.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-javamail.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bsf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-net.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-antlr.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jsch.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-oro.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-logging.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-netrexx.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-testutil.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jdepend.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bcel.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-xalan2.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-resolver.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jmf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-regexp.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-swing.jar:/usr/local/asfpackages/java/jdk1.8.0_144/lib/tools.jar:/x1/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.5.3.jar com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J2-20171024_120805_0103052964008258227603.events @/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J2-20171024_120805_0101988678373599930389.suites -stdin
   [junit4] ERROR: JVM J2 ended with an exception: Forked process returned with error code: 1. Very likely a JVM crash.  See process stdout at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J2-20171024_120805_0106271504522405181204.sysout See process stderr at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J2-20171024_120805_0103589705286221014602.syserr
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1519)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:126)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:982)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:979)
   [junit4] 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4] 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4] 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4] 	at java.lang.Thread.run(Thread.java:748)

BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:834: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:770: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:59: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build.xml:262: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/common-build.xml:556: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/common-build.xml:1494: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/common-build.xml:1021: At least one slave process threw an exception, first: Forked process returned with error code: 1. Very likely a JVM crash.  See process stdout at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J2-20171024_120805_0106271504522405181204.sysout See process stderr at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J2-20171024_120805_0103589705286221014602.syserr

Total time: 1044 minutes 31 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[Fast Archiver] No artifacts from Lucene-Solr-NightlyTests-master #1402 to compare, so performing full copy of artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any

[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1406 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1406/

36 tests failed.
FAILED:  org.apache.lucene.spatial3d.TestGeo3DPoint.testRandomMedium

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
	at __randomizedtesting.SeedInfo.seed([41F452A7E3492FF1]:0)


FAILED:  junit.framework.TestSuite.org.apache.lucene.spatial3d.TestGeo3DPoint

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
	at __randomizedtesting.SeedInfo.seed([41F452A7E3492FF1]:0)


FAILED:  org.apache.solr.cloud.TestAuthenticationFramework.testBasics

Error Message:
Error from server at http://127.0.0.1:41133/solr/testcollection_shard1_replica_n2: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 404 </title> </head> <body> <h2>HTTP ERROR: 404</h2> <p>Problem accessing /solr/testcollection_shard1_replica_n2/update. Reason: <pre>    Can not find: /solr/testcollection_shard1_replica_n2/update</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.20.v20170531</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.CloudSolrClient$RouteException: Error from server at http://127.0.0.1:41133/solr/testcollection_shard1_replica_n2: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 404 </title>
</head>
<body>
<h2>HTTP ERROR: 404</h2>
<p>Problem accessing /solr/testcollection_shard1_replica_n2/update. Reason:
<pre>    Can not find: /solr/testcollection_shard1_replica_n2/update</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.20.v20170531</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([36D99D01EC357CA7:B01332DD4DB22D7]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:541)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:998)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:867)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:800)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
	at org.apache.solr.client.solrj.request.UpdateRequest.commit(UpdateRequest.java:233)
	at org.apache.solr.cloud.TestAuthenticationFramework.collectionCreateSearchDeleteTwice(TestAuthenticationFramework.java:126)
	at org.apache.solr.cloud.TestAuthenticationFramework.testBasics(TestAuthenticationFramework.java:74)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:41133/solr/testcollection_shard1_replica_n2: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 404 </title>
</head>
<body>
<h2>HTTP ERROR: 404</h2>
<p>Problem accessing /solr/testcollection_shard1_replica_n2/update. Reason:
<pre>    Can not find: /solr/testcollection_shard1_replica_n2/update</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.20.v20170531</a><hr/>
</body>
</html>

	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:590)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.lambda$directUpdate$0(CloudSolrClient.java:518)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	... 1 more


FAILED:  org.apache.solr.handler.TestSolrConfigHandlerConcurrent.test

Error Message:
Could not load collection from ZK: collection1

Stack Trace:
org.apache.solr.common.SolrException: Could not load collection from ZK: collection1
	at __randomizedtesting.SeedInfo.seed([36D99D01EC357CA7:BE8DA2DB42C9115F]:0)
	at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1172)
	at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:692)
	at org.apache.solr.common.cloud.ClusterState.getCollectionOrNull(ClusterState.java:130)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.getTotalReplicas(AbstractFullDistribZkTestBase.java:488)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:441)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:334)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:991)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/state.json
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
	at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212)
	at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:357)
	at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:354)
	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
	at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)
	at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1184)
	at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1170)
	... 38 more


FAILED:  org.apache.solr.handler.admin.AutoscalingHistoryHandlerTest.testHistory

Error Message:
expected:<8> but was:<6>

Stack Trace:
java.lang.AssertionError: expected:<8> but was:<6>
	at __randomizedtesting.SeedInfo.seed([36D99D01EC357CA7:5B2539FC567D83A0]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.handler.admin.AutoscalingHistoryHandlerTest.testHistory(AutoscalingHistoryHandlerTest.java:271)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.uninverting.TestDocTermOrds

Error Message:
GC overhead limit exceeded

Stack Trace:
java.lang.OutOfMemoryError: GC overhead limit exceeded
	at __randomizedtesting.SeedInfo.seed([36D99D01EC357CA7]:0)
	at java.lang.Throwable.getStackTraceElement(Native Method)
	at java.lang.Throwable.getOurStackTrace(Throwable.java:827)
	at java.lang.Throwable.getStackTrace(Throwable.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.seedFromThrowable(RandomizedRunner.java:1946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.augmentStackTrace(RandomizedRunner.java:1876)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:945)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.uninverting.TestDocTermOrds

Error Message:
Captured an uncaught exception in thread: Thread[id=96000, name=Lucene Merge Thread #104, state=RUNNABLE, group=TGRP-TestDocTermOrds]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=96000, name=Lucene Merge Thread #104, state=RUNNABLE, group=TGRP-TestDocTermOrds]
Caused by: org.apache.lucene.index.MergePolicy$MergeException: java.lang.OutOfMemoryError: Java heap space
	at __randomizedtesting.SeedInfo.seed([36D99D01EC357CA7]:0)
	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:703)
	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:683)
Caused by: java.lang.OutOfMemoryError: Java heap space
	at org.apache.lucene.store.RAMFile.newBuffer(RAMFile.java:78)
	at org.apache.lucene.store.RAMFile.addBuffer(RAMFile.java:51)
	at org.apache.lucene.store.RAMOutputStream.switchCurrentBuffer(RAMOutputStream.java:164)
	at org.apache.lucene.store.RAMOutputStream.writeBytes(RAMOutputStream.java:150)
	at org.apache.lucene.store.MockIndexOutputWrapper.writeBytes(MockIndexOutputWrapper.java:141)
	at org.apache.lucene.store.RateLimitedIndexOutput.writeBytes(RateLimitedIndexOutput.java:73)
	at org.apache.lucene.codecs.lucene50.Lucene50PostingsWriter.addPosition(Lucene50PostingsWriter.java:291)
	at org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:156)
	at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:864)
	at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:343)
	at org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
	at org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:230)
	at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:115)
	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4390)
	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4030)
	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:624)
	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:661)


FAILED:  org.apache.solr.update.AddBlockUpdateTest.initializationError

Error Message:
GC overhead limit exceeded

Stack Trace:
java.lang.OutOfMemoryError: GC overhead limit exceeded


FAILED:  org.apache.solr.update.AnalysisErrorHandlingTest.initializationError

Error Message:
GC overhead limit exceeded

Stack Trace:
java.lang.OutOfMemoryError: GC overhead limit exceeded


FAILED:  org.apache.solr.update.CdcrUpdateLogTest.initializationError

Error Message:
GC overhead limit exceeded

Stack Trace:
java.lang.OutOfMemoryError: GC overhead limit exceeded


FAILED:  org.apache.solr.update.PeerSyncWithIndexFingerprintCachingTest.initializationError

Error Message:
GC overhead limit exceeded

Stack Trace:
java.lang.OutOfMemoryError: GC overhead limit exceeded


FAILED:  org.apache.solr.update.SolrIndexMetricsTest.initializationError

Error Message:
GC overhead limit exceeded

Stack Trace:
java.lang.OutOfMemoryError: GC overhead limit exceeded


FAILED:  org.apache.solr.update.TestDocBasedVersionConstraints.initializationError

Error Message:
GC overhead limit exceeded

Stack Trace:
java.lang.OutOfMemoryError: GC overhead limit exceeded


FAILED:  org.apache.solr.update.TestExceedMaxTermLength.initializationError

Error Message:
GC overhead limit exceeded

Stack Trace:
java.lang.OutOfMemoryError: GC overhead limit exceeded


FAILED:  org.apache.solr.update.TestHdfsUpdateLog.initializationError

Error Message:
GC overhead limit exceeded

Stack Trace:
java.lang.OutOfMemoryError: GC overhead limit exceeded


FAILED:  org.apache.solr.update.TestInPlaceUpdatesStandalone.initializationError

Error Message:
GC overhead limit exceeded

Stack Trace:
java.lang.OutOfMemoryError: GC overhead limit exceeded


FAILED:  org.apache.solr.update.VersionInfoTest.initializationError

Error Message:
GC overhead limit exceeded

Stack Trace:
java.lang.OutOfMemoryError: GC overhead limit exceeded


FAILED:  org.apache.solr.update.processor.AddSchemaFieldsUpdateProcessorFactoryTest.initializationError

Error Message:
GC overhead limit exceeded

Stack Trace:
java.lang.OutOfMemoryError: GC overhead limit exceeded


FAILED:  org.apache.solr.update.processor.ClassificationUpdateProcessorIntegrationTest.initializationError

Error Message:
GC overhead limit exceeded

Stack Trace:
java.lang.OutOfMemoryError: GC overhead limit exceeded


FAILED:  org.apache.solr.update.processor.ClassificationUpdateProcessorTest.initializationError

Error Message:
GC overhead limit exceeded

Stack Trace:
java.lang.OutOfMemoryError: GC overhead limit exceeded


FAILED:  junit.framework.TestSuite.org.apache.solr.update.processor.CloneFieldUpdateProcessorFactoryTest

Error Message:
GC overhead limit exceeded

Stack Trace:
java.lang.OutOfMemoryError: GC overhead limit exceeded


FAILED:  org.apache.solr.update.processor.IgnoreCommitOptimizeUpdateProcessorFactoryTest.initializationError

Error Message:
GC overhead limit exceeded

Stack Trace:
java.lang.OutOfMemoryError: GC overhead limit exceeded


FAILED:  org.apache.solr.update.processor.ParsingFieldUpdateProcessorsTest.initializationError

Error Message:
GC overhead limit exceeded

Stack Trace:
java.lang.OutOfMemoryError: GC overhead limit exceeded


FAILED:  org.apache.solr.update.processor.PreAnalyzedUpdateProcessorTest.initializationError

Error Message:
GC overhead limit exceeded

Stack Trace:
java.lang.OutOfMemoryError: GC overhead limit exceeded


FAILED:  org.apache.solr.update.processor.SkipExistingDocumentsProcessorFactoryTest.testSkipUpdatesFalseIfInitArgsTrueButFalseBooleanInRequest

Error Message:
 Mockito cannot mock this class: class org.apache.solr.update.processor.DistributedUpdateProcessor.  Mockito can only mock non-private & non-final classes. If you're not sure why you're getting this error, please report to the mailing list.   Java               : 1.8 JVM vendor name    : Oracle Corporation JVM vendor version : 25.144-b01 JVM name           : Java HotSpot(TM) 64-Bit Server VM JVM version        : 1.8.0_144-b01 JVM info           : mixed mode OS name            : Linux OS version         : 4.4.0-83-generic   Underlying exception : java.lang.IllegalArgumentException: Could not create type

Stack Trace:
org.mockito.exceptions.base.MockitoException: 
Mockito cannot mock this class: class org.apache.solr.update.processor.DistributedUpdateProcessor.

Mockito can only mock non-private & non-final classes.
If you're not sure why you're getting this error, please report to the mailing list.


Java               : 1.8
JVM vendor name    : Oracle Corporation
JVM vendor version : 25.144-b01
JVM name           : Java HotSpot(TM) 64-Bit Server VM
JVM version        : 1.8.0_144-b01
JVM info           : mixed mode
OS name            : Linux
OS version         : 4.4.0-83-generic


Underlying exception : java.lang.IllegalArgumentException: Could not create type
	at org.apache.solr.update.processor.SkipExistingDocumentsProcessorFactoryTest.testSkipUpdatesFalseIfInitArgsTrueButFalseBooleanInRequest(SkipExistingDocumentsProcessorFactoryTest.java:180)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:45)
	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:15)
	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:42)
	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:20)
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:263)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:68)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:47)
	at org.junit.runners.ParentRunner$3.run(ParentRunner.java:231)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:60)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:229)
	at org.junit.runners.ParentRunner.access$000(ParentRunner.java:50)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:222)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:300)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:251)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.lang.IllegalArgumentException: Could not create type
	at net.bytebuddy.TypeCache.findOrInsert(TypeCache.java:140)
	at net.bytebuddy.TypeCache$WithInlineExpunction.findOrInsert(TypeCache.java:346)
	at net.bytebuddy.TypeCache.findOrInsert(TypeCache.java:161)
	at net.bytebuddy.TypeCache$WithInlineExpunction.findOrInsert(TypeCache.java:355)
	at org.mockito.internal.creation.bytebuddy.TypeCachingBytecodeGenerator.mockClass(TypeCachingBytecodeGenerator.java:30)
	at org.mockito.internal.creation.bytebuddy.SubclassByteBuddyMockMaker.createMockType(SubclassByteBuddyMockMaker.java:71)
	at org.mockito.internal.creation.bytebuddy.SubclassByteBuddyMockMaker.createMock(SubclassByteBuddyMockMaker.java:42)
	at org.mockito.internal.creation.bytebuddy.ByteBuddyMockMaker.createMock(ByteBuddyMockMaker.java:26)
	at org.mockito.internal.util.MockUtil.createMock(MockUtil.java:35)
	at org.mockito.internal.MockitoCore.mock(MockitoCore.java:65)
	at org.mockito.Mockito.mock(Mockito.java:1681)
	at org.mockito.Mockito.mock(Mockito.java:1594)
	... 21 more
Caused by: java.lang.OutOfMemoryError: GC overhead limit exceeded


FAILED:  org.apache.solr.update.processor.SkipExistingDocumentsProcessorFactoryTest.testSkipUpdatesTrueIfInitArgsFalseButTrueStringInRequest

Error Message:
 Mockito cannot mock this class: class org.apache.solr.update.processor.DistributedUpdateProcessor.  Mockito can only mock non-private & non-final classes. If you're not sure why you're getting this error, please report to the mailing list.   Java               : 1.8 JVM vendor name    : Oracle Corporation JVM vendor version : 25.144-b01 JVM name           : Java HotSpot(TM) 64-Bit Server VM JVM version        : 1.8.0_144-b01 JVM info           : mixed mode OS name            : Linux OS version         : 4.4.0-83-generic   Underlying exception : java.lang.IllegalArgumentException: Could not create type

Stack Trace:
org.mockito.exceptions.base.MockitoException: 
Mockito cannot mock this class: class org.apache.solr.update.processor.DistributedUpdateProcessor.

Mockito can only mock non-private & non-final classes.
If you're not sure why you're getting this error, please report to the mailing list.


Java               : 1.8
JVM vendor name    : Oracle Corporation
JVM vendor version : 25.144-b01
JVM name           : Java HotSpot(TM) 64-Bit Server VM
JVM version        : 1.8.0_144-b01
JVM info           : mixed mode
OS name            : Linux
OS version         : 4.4.0-83-generic


Underlying exception : java.lang.IllegalArgumentException: Could not create type
	at org.apache.solr.update.processor.SkipExistingDocumentsProcessorFactoryTest.testSkipUpdatesTrueIfInitArgsFalseButTrueStringInRequest(SkipExistingDocumentsProcessorFactoryTest.java:197)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:45)
	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:15)
	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:42)
	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:20)
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:263)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:68)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:47)
	at org.junit.runners.ParentRunner$3.run(ParentRunner.java:231)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:60)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:229)
	at org.junit.runners.ParentRunner.access$000(ParentRunner.java:50)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:222)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:300)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:251)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.lang.IllegalArgumentException: Could not create type
	at net.bytebuddy.TypeCache.findOrInsert(TypeCache.java:140)
	at net.bytebuddy.TypeCache$WithInlineExpunction.findOrInsert(TypeCache.java:346)
	at net.bytebuddy.TypeCache.findOrInsert(TypeCache.java:161)
	at net.bytebuddy.TypeCache$WithInlineExpunction.findOrInsert(TypeCache.java:355)
	at org.mockito.internal.creation.bytebuddy.TypeCachingBytecodeGenerator.mockClass(TypeCachingBytecodeGenerator.java:30)
	at org.mockito.internal.creation.bytebuddy.SubclassByteBuddyMockMaker.createMockType(SubclassByteBuddyMockMaker.java:71)
	at org.mockito.internal.creation.bytebuddy.SubclassByteBuddyMockMaker.createMock(SubclassByteBuddyMockMaker.java:42)
	at org.mockito.internal.creation.bytebuddy.ByteBuddyMockMaker.createMock(ByteBuddyMockMaker.java:26)
	at org.mockito.internal.util.MockUtil.createMock(MockUtil.java:35)
	at org.mockito.internal.MockitoCore.mock(MockitoCore.java:65)
	at org.mockito.Mockito.mock(Mockito.java:1681)
	at org.mockito.Mockito.mock(Mockito.java:1594)
	... 21 more
Caused by: java.lang.OutOfMemoryError: GC overhead limit exceeded


FAILED:  org.apache.solr.update.processor.SkipExistingDocumentsProcessorFactoryTest.testSkippableInsertIsNotSkippedIfNotLeader

Error Message:
 Mockito cannot mock this class: class org.apache.solr.update.processor.DistributedUpdateProcessor.  Mockito can only mock non-private & non-final classes. If you're not sure why you're getting this error, please report to the mailing list.   Java               : 1.8 JVM vendor name    : Oracle Corporation JVM vendor version : 25.144-b01 JVM name           : Java HotSpot(TM) 64-Bit Server VM JVM version        : 1.8.0_144-b01 JVM info           : mixed mode OS name            : Linux OS version         : 4.4.0-83-generic   Underlying exception : java.lang.IllegalArgumentException: Could not create type

Stack Trace:
org.mockito.exceptions.base.MockitoException: 
Mockito cannot mock this class: class org.apache.solr.update.processor.DistributedUpdateProcessor.

Mockito can only mock non-private & non-final classes.
If you're not sure why you're getting this error, please report to the mailing list.


Java               : 1.8
JVM vendor name    : Oracle Corporation
JVM vendor version : 25.144-b01
JVM name           : Java HotSpot(TM) 64-Bit Server VM
JVM version        : 1.8.0_144-b01
JVM info           : mixed mode
OS name            : Linux
OS version         : 4.4.0-83-generic


Underlying exception : java.lang.IllegalArgumentException: Could not create type
	at org.apache.solr.update.processor.SkipExistingDocumentsProcessorFactoryTest.testSkippableInsertIsNotSkippedIfNotLeader(SkipExistingDocumentsProcessorFactoryTest.java:209)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:45)
	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:15)
	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:42)
	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:20)
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:263)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:68)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:47)
	at org.junit.runners.ParentRunner$3.run(ParentRunner.java:231)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:60)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:229)
	at org.junit.runners.ParentRunner.access$000(ParentRunner.java:50)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:222)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:300)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:251)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.lang.IllegalArgumentException: Could not create type
	at net.bytebuddy.TypeCache.findOrInsert(TypeCache.java:140)
	at net.bytebuddy.TypeCache$WithInlineExpunction.findOrInsert(TypeCache.java:346)
	at net.bytebuddy.TypeCache.findOrInsert(TypeCache.java:161)
	at net.bytebuddy.TypeCache$WithInlineExpunction.findOrInsert(TypeCache.java:355)
	at org.mockito.internal.creation.bytebuddy.TypeCachingBytecodeGenerator.mockClass(TypeCachingBytecodeGenerator.java:30)
	at org.mockito.internal.creation.bytebuddy.SubclassByteBuddyMockMaker.createMockType(SubclassByteBuddyMockMaker.java:71)
	at org.mockito.internal.creation.bytebuddy.SubclassByteBuddyMockMaker.createMock(SubclassByteBuddyMockMaker.java:42)
	at org.mockito.internal.creation.bytebuddy.ByteBuddyMockMaker.createMock(ByteBuddyMockMaker.java:26)
	at org.mockito.internal.util.MockUtil.createMock(MockUtil.java:35)
	at org.mockito.internal.MockitoCore.mock(MockitoCore.java:65)
	at org.mockito.Mockito.mock(Mockito.java:1681)
	at org.mockito.Mockito.mock(Mockito.java:1594)
	... 21 more
Caused by: java.lang.OutOfMemoryError: GC overhead limit exceeded


FAILED:  org.apache.solr.update.processor.SkipExistingDocumentsProcessorFactoryTest.testSkippableInsertIsNotSkippedIfSkipInsertsFalse

Error Message:
GC overhead limit exceeded

Stack Trace:
java.lang.OutOfMemoryError: GC overhead limit exceeded


FAILED:  org.apache.solr.update.processor.TemplateUpdateProcessorTest.initializationError

Error Message:
GC overhead limit exceeded

Stack Trace:
java.lang.OutOfMemoryError: GC overhead limit exceeded


FAILED:  org.apache.solr.update.processor.TestNamedUpdateProcessors.initializationError

Error Message:
GC overhead limit exceeded

Stack Trace:
java.lang.OutOfMemoryError: GC overhead limit exceeded


FAILED:  org.apache.solr.update.processor.TolerantUpdateProcessorTest.initializationError

Error Message:
GC overhead limit exceeded

Stack Trace:
java.lang.OutOfMemoryError: GC overhead limit exceeded


FAILED:  org.apache.solr.util.TestMaxTokenLenTokenizer.initializationError

Error Message:
GC overhead limit exceeded

Stack Trace:
java.lang.OutOfMemoryError: GC overhead limit exceeded


FAILED:  junit.framework.TestSuite.org.apache.solr.util.TestRTimerTree

Error Message:
GC overhead limit exceeded

Stack Trace:
java.lang.OutOfMemoryError: GC overhead limit exceeded


FAILED:  junit.framework.TestSuite.org.apache.solr.util.TestTestInjection

Error Message:
GC overhead limit exceeded

Stack Trace:
java.lang.OutOfMemoryError: GC overhead limit exceeded


FAILED:  junit.framework.TestSuite.org.apache.solr.util.hll.FullHLLTest

Error Message:
GC overhead limit exceeded

Stack Trace:
java.lang.OutOfMemoryError: GC overhead limit exceeded


FAILED:  junit.framework.TestSuite.org.apache.solr.util.hll.HLLSerializationTest

Error Message:
GC overhead limit exceeded

Stack Trace:
java.lang.OutOfMemoryError: GC overhead limit exceeded




Build Log:
[...truncated 10758 lines...]
   [junit4] Suite: org.apache.lucene.spatial3d.TestGeo3DPoint
   [junit4]   2> okt. 22, 2017 7:33:49 PM com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
   [junit4]   2> WARNING: Suite execution timed out: org.apache.lucene.spatial3d.TestGeo3DPoint
   [junit4]   2>    1) Thread[id=9, name=JUnit4-serializer-daemon, state=TIMED_WAITING, group=main]
   [junit4]   2>         at java.lang.Thread.sleep(Native Method)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:50)
   [junit4]   2>    2) Thread[id=13, name=SUITE-TestGeo3DPoint-seed#[41F452A7E3492FF1], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
   [junit4]   2>         at java.lang.Thread.getStackTrace(Thread.java:1559)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:696)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:693)
   [junit4]   2>         at java.security.AccessController.doPrivileged(Native Method)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getStackTrace(ThreadLeakControl.java:693)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getThreadsWithTraces(ThreadLeakControl.java:709)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.formatThreadStacksFull(ThreadLeakControl.java:689)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.access$1000(ThreadLeakControl.java:65)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$2.evaluate(ThreadLeakControl.java:415)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:705)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.access$200(RandomizedRunner.java:139)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$2.run(RandomizedRunner.java:626)
   [junit4]   2>    3) Thread[id=1, name=main, state=WAITING, group=main]
   [junit4]   2>         at java.lang.Object.wait(Native Method)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1252)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1326)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:636)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:493)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:251)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
   [junit4]   2>    4) Thread[id=14, name=TEST-TestGeo3DPoint.testRandomMedium-seed#[41F452A7E3492FF1], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoComplexPolygon$Node.traverse(GeoComplexPolygon.java:561)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoComplexPolygon$Node.traverse(GeoComplexPolygon.java:561)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoComplexPolygon$Tree.traverse(GeoComplexPolygon.java:649)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoComplexPolygon$Tree.traverse(GeoComplexPolygon.java:635)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoComplexPolygon.isWithin(GeoComplexPolygon.java:367)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoBaseMembershipShape.isWithin(GeoBaseMembershipShape.java:36)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoBaseShape.getBounds(GeoBaseShape.java:43)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoComplexPolygon.getBounds(GeoComplexPolygon.java:441)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.verify(TestGeo3DPoint.java:846)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.doTestRandom(TestGeo3DPoint.java:548)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.testRandomMedium(TestGeo3DPoint.java:475)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestGeo3DPoint -Dtests.method=testRandomMedium -Dtests.seed=41F452A7E3492FF1 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=lv-LV -Dtests.timezone=America/Indiana/Knox -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR    455s J1 | TestGeo3DPoint.testRandomMedium <<<
   [junit4]    > Throwable #1: java.lang.Exception: Test abandoned because suite timeout was reached.
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([41F452A7E3492FF1]:0)
   [junit4]   2> okt. 22, 2017 7:33:50 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> okt. 22, 2017 7:34:10 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 1 thread leaked from SUITE scope at org.apache.lucene.spatial3d.TestGeo3DPoint: 
   [junit4]   2>    1) Thread[id=14, name=TEST-TestGeo3DPoint.testRandomMedium-seed#[41F452A7E3492FF1], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.Plane.findIntersectionBounds(Plane.java:1073)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.Plane.recordBounds(Plane.java:1214)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.XYZBounds.addIntersection(XYZBounds.java:262)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoConvexPolygon.getBounds(GeoConvexPolygon.java:486)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoBaseCompositeShape.getBounds(GeoBaseCompositeShape.java:131)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.verify(TestGeo3DPoint.java:846)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.doTestRandom(TestGeo3DPoint.java:548)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.testRandomMedium(TestGeo3DPoint.java:475)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> okt. 22, 2017 7:34:10 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=14, name=TEST-TestGeo3DPoint.testRandomMedium-seed#[41F452A7E3492FF1], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
   [junit4]   2> okt. 22, 2017 7:34:13 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> SEVERE: There are still zombie threads that couldn't be terminated:
   [junit4]   2>    1) Thread[id=14, name=TEST-TestGeo3DPoint.testRandomMedium-seed#[41F452A7E3492FF1], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
   [junit4]   2>         at java.lang.Long.valueOf(Long.java:840)
   [junit4]   2>         at org.junit.Assert.assertEquals(Assert.java:472)
   [junit4]   2>         at org.junit.Assert.assertEquals(Assert.java:456)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.verify(TestGeo3DPoint.java:839)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.doTestRandom(TestGeo3DPoint.java:548)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.testRandomMedium(TestGeo3DPoint.java:475)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarity(queryNorm=true): {}, locale=lv-LV, timezone=America/Indiana/Knox
   [junit4]   2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=2,free=153785112,total=489160704
   [junit4]   2> NOTE: All tests run in this JVM: [TestGeo3DDocValues, TestGeo3DPoint]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestGeo3DPoint -Dtests.seed=41F452A7E3492FF1 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=lv-LV -Dtests.timezone=America/Indiana/Knox -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   0.00s J1 | TestGeo3DPoint (suite) <<<
   [junit4]    > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([41F452A7E3492FF1]:0)
   [junit4] Completed [15/15 (1!)] on J1 in 7223.76s, 9 tests, 2 errors <<< FAILURES!

[...truncated 1959 lines...]
   [junit4] Suite: org.apache.solr.handler.admin.AutoscalingHistoryHandlerTest
   [junit4]   2> 831121 INFO  (SUITE-AutoscalingHistoryHandlerTest-seed#[36D99D01EC357CA7]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.admin.AutoscalingHistoryHandlerTest_36D99D01EC357CA7-001/init-core-data-001
   [junit4]   2> 831122 WARN  (SUITE-AutoscalingHistoryHandlerTest-seed#[36D99D01EC357CA7]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=5 numCloses=5
   [junit4]   2> 831123 INFO  (SUITE-AutoscalingHistoryHandlerTest-seed#[36D99D01EC357CA7]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 831124 INFO  (SUITE-AutoscalingHistoryHandlerTest-seed#[36D99D01EC357CA7]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 831162 INFO  (SUITE-AutoscalingHistoryHandlerTest-seed#[36D99D01EC357CA7]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 3 servers in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.admin.AutoscalingHistoryHandlerTest_36D99D01EC357CA7-001/tempDir-001
   [junit4]   2> 831162 INFO  (SUITE-AutoscalingHistoryHandlerTest-seed#[36D99D01EC357CA7]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 831178 INFO  (Thread-2176) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 831178 INFO  (Thread-2176) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 831239 ERROR (Thread-2176) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 831279 INFO  (SUITE-AutoscalingHistoryHandlerTest-seed#[36D99D01EC357CA7]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:35449
   [junit4]   2> 831402 INFO  (jetty-launcher-850-thread-1) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 831422 INFO  (jetty-launcher-850-thread-2) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 831423 INFO  (jetty-launcher-850-thread-3) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 831467 INFO  (jetty-launcher-850-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2ced7057{/solr,null,AVAILABLE}
   [junit4]   2> 831468 INFO  (jetty-launcher-850-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@30c9991d{SSL,[ssl, http/1.1]}{127.0.0.1:39920}
   [junit4]   2> 831468 INFO  (jetty-launcher-850-thread-1) [    ] o.e.j.s.Server Started @844427ms
   [junit4]   2> 831468 INFO  (jetty-launcher-850-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=39920}
   [junit4]   2> 831468 ERROR (jetty-launcher-850-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 831468 INFO  (jetty-launcher-850-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 831468 INFO  (jetty-launcher-850-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 831468 INFO  (jetty-launcher-850-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 831468 INFO  (jetty-launcher-850-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-23T00:58:03.497Z
   [junit4]   2> 831532 INFO  (jetty-launcher-850-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1abb0b51{/solr,null,AVAILABLE}
   [junit4]   2> 831532 INFO  (jetty-launcher-850-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@3c6479e0{SSL,[ssl, http/1.1]}{127.0.0.1:40313}
   [junit4]   2> 831532 INFO  (jetty-launcher-850-thread-2) [    ] o.e.j.s.Server Started @844491ms
   [junit4]   2> 831532 INFO  (jetty-launcher-850-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=40313}
   [junit4]   2> 831532 ERROR (jetty-launcher-850-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 831532 INFO  (jetty-launcher-850-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 831532 INFO  (jetty-launcher-850-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 831532 INFO  (jetty-launcher-850-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 831532 INFO  (jetty-launcher-850-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-23T00:58:03.561Z
   [junit4]   2> 831604 INFO  (jetty-launcher-850-thread-3) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@e5d7a7d{/solr,null,AVAILABLE}
   [junit4]   2> 831605 INFO  (jetty-launcher-850-thread-3) [    ] o.e.j.s.AbstractConnector Started ServerConnector@6f5a0197{SSL,[ssl, http/1.1]}{127.0.0.1:46269}
   [junit4]   2> 831605 INFO  (jetty-launcher-850-thread-3) [    ] o.e.j.s.Server Started @844564ms
   [junit4]   2> 831605 INFO  (jetty-launcher-850-thread-3) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=46269}
   [junit4]   2> 831605 ERROR (jetty-launcher-850-thread-3) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 831605 INFO  (jetty-launcher-850-thread-3) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 831605 INFO  (jetty-launcher-850-thread-3) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 831605 INFO  (jetty-launcher-850-thread-3) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 831605 INFO  (jetty-launcher-850-thread-3) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-23T00:58:03.634Z
   [junit4]   2> 831613 INFO  (jetty-launcher-850-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 831725 INFO  (jetty-launcher-850-thread-3) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 831885 INFO  (jetty-launcher-850-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 831954 INFO  (jetty-launcher-850-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:35449/solr
   [junit4]   2> 832019 INFO  (jetty-launcher-850-thread-3) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:35449/solr
   [junit4]   2> 832175 INFO  (jetty-launcher-850-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:35449/solr
   [junit4]   2> 832873 INFO  (jetty-launcher-850-thread-2) [n:127.0.0.1:40313_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 832874 INFO  (jetty-launcher-850-thread-2) [n:127.0.0.1:40313_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:40313_solr
   [junit4]   2> 832874 INFO  (jetty-launcher-850-thread-2) [n:127.0.0.1:40313_solr    ] o.a.s.c.Overseer Overseer (id=98875492481564681-127.0.0.1:40313_solr-n_0000000000) starting
   [junit4]   2> 832961 DEBUG (OverseerStateUpdate-98875492481564681-127.0.0.1:40313_solr-n_0000000000) [n:127.0.0.1:40313_solr    ] o.a.s.c.Overseer Starting to work on the main queue
   [junit4]   2> 832962 DEBUG (OverseerAutoScalingTriggerThread-98875492481564681-127.0.0.1:40313_solr-n_0000000000) [n:127.0.0.1:40313_solr    ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 832961 INFO  (jetty-launcher-850-thread-2) [n:127.0.0.1:40313_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40313_solr
   [junit4]   2> 833033 DEBUG (OverseerAutoScalingTriggerThread-98875492481564681-127.0.0.1:40313_solr-n_0000000000) [n:127.0.0.1:40313_solr    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
   [junit4]   2> 833045 INFO  (OverseerStateUpdate-98875492481564681-127.0.0.1:40313_solr-n_0000000000) [n:127.0.0.1:40313_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 833070 DEBUG (OverseerAutoScalingTriggerThread-98875492481564681-127.0.0.1:40313_solr-n_0000000000) [n:127.0.0.1:40313_solr    ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:40313_solr]
   [junit4]   2> 833106 DEBUG (OverseerAutoScalingTriggerThread-98875492481564681-127.0.0.1:40313_solr-n_0000000000) [n:127.0.0.1:40313_solr    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion -1
   [junit4]   2> 833106 DEBUG (OverseerAutoScalingTriggerThread-98875492481564681-127.0.0.1:40313_solr-n_0000000000) [n:127.0.0.1:40313_solr    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
   [junit4]   2> 833122 DEBUG (OverseerAutoScalingTriggerThread-98875492481564681-127.0.0.1:40313_solr-n_0000000000) [n:127.0.0.1:40313_solr    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 833123 DEBUG (OverseerAutoScalingTriggerThread-98875492481564681-127.0.0.1:40313_solr-n_0000000000) [n:127.0.0.1:40313_solr    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
   [junit4]   2> 833195 DEBUG (ScheduledTrigger-1112-thread-1) [n:127.0.0.1:40313_solr    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:40313_solr]
   [junit4]   2> 833414 INFO  (jetty-launcher-850-thread-3) [n:127.0.0.1:46269_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 833448 INFO  (jetty-launcher-850-thread-3) [n:127.0.0.1:46269_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 833449 INFO  (jetty-launcher-850-thread-3) [n:127.0.0.1:46269_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:46269_solr
   [junit4]   2> 833453 INFO  (jetty-launcher-850-thread-2) [n:127.0.0.1:40313_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40313.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@71798be9
   [junit4]   2> 833552 INFO  (jetty-launcher-850-thread-1) [n:127.0.0.1:39920_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 833573 INFO  (zkCallback-866-thread-1-processing-n:127.0.0.1:46269_solr) [n:127.0.0.1:46269_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 833574 INFO  (jetty-launcher-850-thread-1) [n:127.0.0.1:39920_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 833602 INFO  (jetty-launcher-850-thread-1) [n:127.0.0.1:39920_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:39920_solr
   [junit4]   2> 833602 INFO  (zkCallback-866-thread-1-processing-n:127.0.0.1:46269_solr) [n:127.0.0.1:46269_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 833615 INFO  (jetty-launcher-850-thread-2) [n:127.0.0.1:40313_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40313.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@71798be9
   [junit4]   2> 833615 INFO  (jetty-launcher-850-thread-2) [n:127.0.0.1:40313_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40313.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@71798be9
   [junit4]   2> 833616 INFO  (jetty-launcher-850-thread-2) [n:127.0.0.1:40313_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.admin.AutoscalingHistoryHandlerTest_36D99D01EC357CA7-001/tempDir-001/node2/.
   [junit4]   2> 833671 DEBUG (OverseerStateUpdate-98875492481564681-127.0.0.1:40313_solr-n_0000000000) [n:127.0.0.1:40313_solr    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "operation":"downnode",
   [junit4]   2>   "node_name":"127.0.0.1:46269_solr"} current state version: 0
   [junit4]   2> 833671 DEBUG (OverseerStateUpdate-98875492481564681-127.0.0.1:40313_solr-n_0000000000) [n:127.0.0.1:40313_solr    ] o.a.s.c.o.NodeMutator DownNode state invoked for node: 127.0.0.1:46269_solr
   [junit4]   2> 833687 INFO  (zkCallback-869-thread-2-processing-n:127.0.0.1:40313_solr) [n:127.0.0.1:40313_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 833708 INFO  (zkCallback-869-thread-3-processing-n:127.0.0.1:40313_solr) [n:127.0.0.1:40313_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 833708 DEBUG (OverseerStateUpdate-98875492481564681-127.0.0.1:40313_solr-n_0000000000) [n:127.0.0.1:40313_solr    ] o.a.s.c.Overseer processMessage: queueSize: 2, message = {
   [junit4]   2>   "operation":"downnode",
   [junit4]   2>   "node_name":"127.0.0.1:39920_solr"} current state version: 0
   [junit4]   2> 833708 DEBUG (OverseerStateUpdate-98875492481564681-127.0.0.1:40313_solr-n_0000000000) [n:127.0.0.1:40313_solr    ] o.a.s.c.o.NodeMutator DownNode state invoked for node: 127.0.0.1:39920_solr
   [junit4]   2> 833781 INFO  (zkCallback-861-thread-1-processing-n:127.0.0.1:39920_solr) [n:127.0.0.1:39920_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 834295 DEBUG (ScheduledTrigger-1112-thread-1) [n:127.0.0.1:40313_solr    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:39920_solr, 127.0.0.1:40313_solr, 127.0.0.1:46269_solr]
   [junit4]   2> 834410 INFO  (jetty-launcher-850-thread-3) [n:127.0.0.1:46269_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_46269.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@71798be9
   [junit4]   2> 834419 INFO  (jetty-launcher-850-thread-1) [n:127.0.0.1:39920_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_39920.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@71798be9
   [junit4]   2> 834437 INFO  (jetty-launcher-850-thread-3) [n:127.0.0.1:46269_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_46269.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@71798be9
   [junit4]   2> 834437 INFO  (jetty-launcher-850-thread-3) [n:127.0.0.1:46269_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_46269.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@71798be9
   [junit4]   2> 834438 INFO  (jetty-launcher-850-thread-3) [n:127.0.0.1:46269_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.admin.AutoscalingHistoryHandlerTest_36D99D01EC357CA7-001/tempDir-001/node3/.
   [junit4]   2> 834457 INFO  (jetty-launcher-850-thread-1) [n:127.0.0.1:39920_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_39920.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@71798be9
   [junit4]   2> 834457 INFO  (jetty-launcher-850-thread-1) [n:127.0.0.1:39920_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_39920.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@71798be9
   [junit4]   2> 834458 INFO  (jetty-launcher-850-thread-1) [n:127.0.0.1:39920_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.admin.AutoscalingHistoryHandlerTest_36D99D01EC357CA7-001/tempDir-001/node1/.
   [junit4]   2> 834841 INFO  (SUITE-AutoscalingHistoryHandlerTest-seed#[36D99D01EC357CA7]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 834931 INFO  (SUITE-AutoscalingHistoryHandlerTest-seed#[36D99D01EC357CA7]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:35449/solr ready
   [junit4]   2> 835216 INFO  (qtp1784054990-5182) [n:127.0.0.1:39920_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=3&name=AutoscalingHistoryHandlerTest_collection&nrtReplicas=3&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 835241 INFO  (OverseerThreadFactory-1114-thread-1-processing-n:127.0.0.1:40313_solr) [n:127.0.0.1:40313_solr    ] o.a.s.c.CreateCollectionCmd Create collection AutoscalingHistoryHandlerTest_collection
   [junit4]   2> 835242 INFO  (OverseerThreadFactory-1114-thread-1-processing-n:127.0.0.1:40313_solr) [n:127.0.0.1:40313_solr    ] o.a.s.c.CreateCollectionCmd Only _default config set found, using it.
   [junit4]   2> 835242 INFO  (OverseerThreadFactory-1114-thread-1-processing-n:127.0.0.1:40313_solr) [n:127.0.0.1:40313_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/protwords.txt to /configs/AutoscalingHistoryHandlerTest_collection/protwords.txt
   [junit4]   2> 835256 INFO  (OverseerThreadFactory-1114-thread-1-processing-n:127.0.0.1:40313_solr) [n:127.0.0.1:40313_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/managed-schema to /configs/AutoscalingHistoryHandlerTest_collection/managed-schema
   [junit4]   2> 835258 INFO  (OverseerThreadFactory-1114-thread-1-processing-n:127.0.0.1:40313_solr) [n:127.0.0.1:40313_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/solrconfig.xml to /configs/AutoscalingHistoryHandlerTest_collection/solrconfig.xml
   [junit4]   2> 835259 INFO  (OverseerThreadFactory-1114-thread-1-processing-n:127.0.0.1:40313_solr) [n:127.0.0.1:40313_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/synonyms.txt to /configs/AutoscalingHistoryHandlerTest_collection/synonyms.txt
   [junit4]   2> 835281 INFO  (OverseerThreadFactory-1114-thread-1-processing-n:127.0.0.1:40313_solr) [n:127.0.0.1:40313_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/stopwords.txt to /configs/AutoscalingHistoryHandlerTest_collection/stopwords.txt
   [junit4]   2> 835281 INFO  (OverseerThreadFactory-1114-thread-1-processing-n:127.0.0.1:40313_solr) [n:127.0.0.1:40313_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/params.json to /configs/AutoscalingHistoryHandlerTest_collection/params.json
   [junit4]   2> 835282 INFO  (OverseerThreadFactory-1114-thread-1-processing-n:127.0.0.1:40313_solr) [n:127.0.0.1:40313_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/contractions_it.txt to /configs/AutoscalingHistoryHandlerTest_collection/lang/contractions_it.txt
   [junit4]   2> 835283 INFO  (OverseerThreadFactory-1114-thread-1-processing-n:127.0.0.1:40313_solr) [n:127.0.0.1:40313_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_no.txt to /configs/AutoscalingHistoryHandlerTest_collection/lang/stopwords_no.txt
   [junit4]   2> 835350 INFO  (OverseerThreadFactory-1114-thread-1-processing-n:127.0.0.1:40313_solr) [n:127.0.0.1:40313_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_hy.txt to /configs/AutoscalingHistoryHandlerTest_collection/lang/stopwords_hy.txt
   [junit4]   2> 835352 INFO  (OverseerThreadFactory-1114-thread-1-processing-n:127.0.0.1:40313_solr) [n:127.0.0.1:40313_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stemdict_nl.txt to /configs/AutoscalingHistoryHandlerTest_collection/lang/stemdict_nl.txt
   [junit4]   2> 835353 INFO  (OverseerThreadFactory-1114-thread-1-processing-n:127.0.0.1:40313_solr) [n:127.0.0.1:40313_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/contractions_ca.txt to /configs/AutoscalingHistoryHandlerTest_collection/lang/contractions_ca.txt
   [junit4]   2> 835354 INFO  (OverseerThreadFactory-1114-thread-1-processing-n:127.0.0.1:40313_solr) [n:127.0.0.1:40313_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_id.txt to /configs/AutoscalingHistoryHandlerTest_collection/lang/stopwords_id.txt
   [junit4]   2> 835410 INFO  (OverseerThreadFactory-1114-thread-1-processing-n:127.0.0.1:40313_solr) [n:127.0.0.1:40313_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_ga.txt to /configs/AutoscalingHistoryHandlerTest_collection/lang/stopwords_ga.txt
   [junit4]   2> 835448 DEBUG (ScheduledTrigger-1112-thread-2) [n:127.0.0.1:40313_solr    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:39920_solr, 127.0.0.1:40313_solr, 127.0.0.1:46269_solr]
   [junit4]   2> 835468 INFO  (OverseerThreadFactory-1114-thread-1-processing-n:127.0.0.1:40313_solr) [n:127.0.0.1:40313_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_da.txt to /configs/AutoscalingHistoryHandlerTest_collection/lang/stopwords_da.txt
   [junit4]   2> 835469 INFO  (OverseerThreadFactory-1114-thread-1-processing-n:127.0.0.1:40313_solr) [n:127.0.0.1:40313_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_hi.txt to /configs/AutoscalingHistoryHandlerTest_collection/lang/stopwords_hi.txt
   [junit4]   2> 835470 INFO  (OverseerThreadFactory-1114-thread-1-processing-n:127.0.0.1:40313_solr) [n:127.0.0.1:40313_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_ja.txt to /configs/AutoscalingHistoryHandlerTest_collection/lang/stopwords_ja.txt
   [junit4]   2> 835471 INFO  (OverseerThreadFactory-1114-thread-1-processing-n:127.0.0.1:40313_solr) [n:127.0.0.1:40313_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_pt.txt to /configs/AutoscalingHistoryHandlerTest_collection/lang/stopwords_pt.txt
   [junit4]   2> 835484 INFO  (OverseerThreadFactory-1114-thread-1-processing-n:127.0.0.1:40313_solr) [n:127.0.0.1:40313_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_hu.txt to /configs/AutoscalingHistoryHandlerTest_collection/lang/stopwords_hu.txt
   [junit4]   2> 835485 INFO  (OverseerThreadFactory-1114-thread-1-processing-n:127.0.0.1:40313_solr) [n:127.0.0.1:40313_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_el.txt to /configs/AutoscalingHistoryHandlerTest_collection/lang/stopwords_el.txt
   [junit4]   2> 835486 INFO  (OverseerThreadFactory-1114-thread-1-processing-n:127.0.0.1:40313_solr) [n:127.0.0.1:40313_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_ru.txt to /configs/AutoscalingHistoryHandlerTest_collection/lang/stopwords_ru.txt
   [junit4]   2> 835486 INFO  (OverseerThreadFactory-1114-thread-1-processing-n:127.0.0.1:40313_solr) [n:127.0.0.1:40313_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_tr.txt to /configs/AutoscalingHistoryHandlerTest_collection/lang/stopwords_tr.txt
   [junit4]   2> 835487 INFO  (OverseerThreadFactory-1114-thread-1-processing-n:127.0.0.1:40313_solr) [n:127.0.0.1:40313_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_ar.txt to /configs/AutoscalingHistoryHandlerTest_collection/lang/stopwords_ar.txt
   [junit4]   2> 835520 INFO  (OverseerThreadFactory-1114-thread-1-processing-n:127.0.0.1:40313_solr) [n:127.0.0.1:40313_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/userdict_ja.txt to /configs/AutoscalingHistoryHandlerTest_collection/lang/userdict_ja.txt
   [junit4]   2> 835522 INFO  (OverseerThreadFactory-1114-thread-1-processing-n:127.0.0.1:40313_solr) [n:127.0.0.1:40313_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_it.txt to /configs/AutoscalingHistoryHandlerTest_collection/lang/stopwords_it.txt
   [junit4]   2> 835523 INFO  (OverseerThreadFactory-1114-thread-1-processing-n:127.0.0.1:40313_solr) [n:127.0.0.1:40313_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_eu.txt to /configs/AutoscalingHistoryHandlerTest_collection/lang/stopwords_eu.txt
   [junit4]   2> 835524 INFO  (OverseerThreadFactory-1114-thread-1-processing-n:127.0.0.1:40313_solr) [n:127.0.0.1:40313_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_cz.txt to /configs/AutoscalingHistoryHandlerTest_collection/lang/stopwords_cz.txt
   [junit4]   2> 835557 INFO  (OverseerThreadFactory-1114-thread-1-processing-n:127.0.0.1:40313_solr) [n:127.0.0.1:40313_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stoptags_ja.txt to /configs/AutoscalingHistoryHandlerTest_collection/lang/stoptags_ja.txt
   [junit4]   2> 835558 INFO  (OverseerThreadFactory-1114-thread-1-processing-n:127.0.0.1:40313_solr) [n:127.0.0.1:40313_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/contractions_fr.txt to /configs/AutoscalingHistoryHandlerTest_collection/lang/contractions_fr.txt
   [junit4]   2> 835559 INFO  (OverseerThreadFactory-1114-thread-1-processing-n:127.0.0.1:40313_solr) [n:127.0.0.1:40313_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_fa.txt to /configs/AutoscalingHistoryHandlerTest_collection/lang/stopwords_fa.txt
   [junit4]   2> 835559 INFO  (OverseerThreadFactory-1114-thread-1-processing-n:127.0.0.1:40313_solr) [n:127.0.0.1:40313_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_de.txt to /configs/AutoscalingHistoryHandlerTest_collection/lang/stopwords_de.txt
   [junit4]   2> 835560 INFO  (OverseerThreadFactory-1114-thread-1-processing-n:127.0.0.1:40313_solr) [n:127.0.0.1:40313_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/contractions_ga.txt to /configs/AutoscalingHistoryHandlerTest_collection/lang/contractions_ga.txt
   [junit4]   2> 835561 INFO  (OverseerThreadFactory-1114-thread-1-processing-n:127.0.0.1:40313_solr) [n:127.0.0.1:40313_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_ca.txt to /configs/AutoscalingHistoryHandlerTest_collection/lang/stopwords_ca.txt
   [junit4]   2> 835580 INFO  (OverseerThreadFactory-1114-thread-1-processing-n:127.0.0.1:40313_solr) [n:127.0.0.1:40313_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_nl.txt to /configs/AutoscalingHistoryHandlerTest_collection/lang/stopwords_nl.txt
   [junit4]   2> 835581 INFO  (OverseerThreadFactory-1114-thread-1-processing-n:127.0.0.1:40313_solr) [n:127.0.0.1:40313_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default

[...truncated too long message...]

iddenPrefixQueryForCustomFieldType, TestQueryWrapperFilter, TestRecoveryHdfs, TestOrdValues, TestSortByMinMaxFunction, BlockJoinFacetDistribTest, TestCloudNestedDocsSort, TestExactStatsCache, PKIAuthenticationIntegrationTest, TestAnalyzeInfixSuggestions, TestBlendedInfixSuggestions, TestFreeTextSuggestions, BlockCacheTest, BlockDirectoryTest, TestDocTermOrds, CloneFieldUpdateProcessorFactoryTest, TestRTimerTree, TestTestInjection, FullHLLTest]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=FullHLLTest -Dtests.seed=36D99D01EC357CA7 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=zh-SG -Dtests.timezone=America/Kralendijk -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   0.00s J0 | FullHLLTest (suite) <<<
   [junit4]    > Throwable #1: java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4] Completed [742/745 (29!)] on J0 in 2.50s, 0 tests, 1 error <<< FAILURES!

[...truncated 2 lines...]
   [junit4] Suite: org.apache.solr.util.hll.HLLSerializationTest
   [junit4]   2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarity(queryNorm=true): {field=IB SPL-DZ(0.3)}, locale=zh-SG, timezone=America/Kralendijk
   [junit4]   2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=60163728,total=477626368
   [junit4]   2> NOTE: All tests run in this JVM: [CleanupOldIndexTest, TestLuceneIndexBackCompat, HdfsWriteToMultipleCollectionsTest, ZkStateReaderTest, TestMergePolicyConfig, TestSubQueryTransformerCrossCore, TestConfigSetsAPI, SubstringBytesRefFilterTest, MissingSegmentRecoveryTest, OutOfBoxZkACLAndCredentialsProvidersTest, CdcrReplicationHandlerTest, DistributedMLTComponentTest, TestSchemaSimilarityResource, DistanceUnitsTest, TestLRUStatsCache, TestLeaderElectionWithEmptyReplica, AutoscalingHistoryHandlerTest, HdfsDirectoryFactoryTest, TestPerFieldSimilarityWithDefaultOverride, HdfsRecoveryZkTest, TestSolrIndexConfig, OverseerTaskQueueTest, BaseCdcrDistributedZkTest, SpellCheckCollatorWithCollapseTest, TestInitParams, SyncSliceTest, MetricsConfigTest, SearchHandlerTest, TestMultiValuedNumericRangeQuery, CursorMarkTest, TestPHPSerializedResponseWriter, RequestLoggingTest, SpatialFilterTest, SoftAutoCommitTest, HdfsThreadLeakTest, TestDownShardTolerantSearch, CoreSorterTest, TestFileDictionaryLookup, TestPolicyCloud, TestShardHandlerFactory, BlockJoinFacetSimpleTest, TestStressUserVersions, TestRandomFaceting, BigEndianAscendingWordSerializerTest, TestDistribIDF, DocExpirationUpdateProcessorFactoryTest, ZkFailoverTest, TestStreamBody, TestClusterProperties, DocValuesMissingTest, ExternalFileFieldSortTest, BJQParserTest, SaslZkACLProviderTest, AtomicUpdateProcessorFactoryTest, TestCollationFieldDocValues, TestGraphTermsQParserPlugin, DirectUpdateHandlerOptimizeTest, TestImpersonationWithHadoopAuth, URLClassifyProcessorTest, TestSolrJ, TestReqParamsAPI, ConnectionManagerTest, TestDynamicLoading, LeaderElectionIntegrationTest, TestMinMaxOnMultiValuedField, JsonLoaderTest, TestCrossCoreJoin, TestSolrQueryResponse, TestRebalanceLeaders, TestSuggestSpellingConverter, ResponseLogComponentTest, TestConfigSets, LeaderElectionContextKeyTest, TestStressVersions, TestMultiWordSynonyms, BasicZkTest, UtilsToolTest, TestRawResponseWriter, DistributedFacetPivotLongTailTest, SolrXmlInZkTest, RAMDirectoryFactoryTest, TestJsonRequest, TestUninvertingReader, RuleEngineTest, TestCSVLoader, SimpleCollectionCreateDeleteTest, TestRandomCollapseQParserPlugin, DocumentBuilderTest, SolrSlf4jReporterTest, HttpSolrCallGetCoreTest, HdfsAutoAddReplicasIntegrationTest, TestConfig, TestExpandComponent, AnalysisAfterCoreReloadTest, TestConfigOverlay, TestPhraseSuggestions, TestPayloadCheckQParserPlugin, SolrMetricReporterTest, DeleteReplicaTest, TestBadConfig, IndexSchemaRuntimeFieldTest, TestSolrCoreParser, TestSystemCollAutoCreate, TestIndexingPerformance, JSONWriterTest, SparseHLLTest, SpellCheckComponentTest, TestUtils, SolrPluginUtilsTest, TestComponentsName, UpdateLogTest, FileUtilsTest, TestSolrXml, TestCustomDocTransformer, SliceStateTest, TestLegacyFieldReuse, TestCustomSort, TestQueryUtils, RemoteQueryErrorTest, TestSweetSpotSimilarityFactory, TestIBSimilarityFactory, PreAnalyzedFieldManagedSchemaCloudTest, TestFastOutputStream, RankQueryTest, BasicDistributedZkTest, ChaosMonkeySafeLeaderTest, TestReplicationHandler, ClusterStateUpdateTest, ZkSolrClientTest, TestRandomDVFaceting, ZkCLITest, ShardRoutingCustomTest, TestDistributedSearch, TestDistributedGrouping, TestFaceting, TestRecovery, DistributedSpellCheckComponentTest, ZkControllerTest, TestJoin, TestRangeQuery, SimpleFacetsTest, TestSolr4Spatial, SolrCoreTest, SolrCmdDistributorTest, QueryElevationComponentTest, PeerSyncTest, BadIndexSchemaTest, TestSort, BasicFunctionalityTest, TestLazyCores, DirectUpdateHandlerTest, HighlighterTest, ShowFileRequestHandlerTest, DistributedQueryElevationComponentTest, SolrIndexSplitterTest, SimplePostToolTest, TestCoreDiscovery, DocValuesTest, TestWriterPerf, PrimitiveFieldTypeTest, TestOmitPositions, TermsComponentTest, QueryParsingTest, TestLFUCache, ChaosMonkeyNothingIsSafeWithPullReplicasTest, CollectionsAPISolrJTest, DistributedQueueTest, DistributedVersionInfoTest, HttpPartitionTest, MoveReplicaTest, ReplicationFactorTest, RestartWhileUpdatingTest, TestRequestForwarding, TestRequestStatusCollectionAPI, TestSizeLimitedDistributedMap, TestStressCloudBlindAtomicUpdates, HdfsNNFailoverTest, HdfsTlogReplayBufferedWhileIndexingTest, TestSQLHandlerNonCloud, V2ApiIntegrationTest, CoreAdminCreateDiscoverTest, CoreMergeIndexesAdminHandlerTest, PropertiesRequestHandlerTest, SegmentsInfoRequestHandlerTest, StatsReloadRaceTest, TestApiFramework, TestCollectionAPIs, TestConfigsApi, TestCoreAdminApis, DistributedDebugComponentTest, DistributedExpandComponentTest, DistributedFacetExistsSmallTest, DistributedFacetPivotWhiteBoxTest, DistributedQueryComponentOptimizationTest, FacetPivotSmallTest, InfixSuggestersTest, ReplicaListTransformerTest, ShufflingReplicaListTransformerTest, SpatialHeatmapFacetsTest, TestPivotHelperCode, TestTrackingShardHandlerFactory, HighlighterMaxOffsetTest, TestUnifiedSolrHighlighter, TestSlowCompositeReaderWrapper, UninvertDocValuesMergePolicyTest, WrapperMergePolicyFactoryTest, CheckHdfsIndexTest, TestLegacyField, TestLegacyNumericUtils, TestLegacyTerms, TestNumericRangeQuery32, TestNumericTokenStream, SolrGangliaReporterTest, SolrGraphiteReporterTest, SolrJmxReporterCloudTest, SolrJmxReporterTest, TestBinaryResponseWriter, TestGraphMLResponseWriter, TestPushWriter, TestRawTransformer, TestSubQueryTransformer, TestManagedResource, TestManagedResourceStorage, TestCopyFieldCollectionResource, TestDynamicFieldResource, TestFieldCollectionResource, TestFieldResource, TestFieldTypeResource, TestSchemaNameResource, TestSchemaVersionResource, TestSerializedLuceneMatchVersion, TestUniqueKeyFieldResource, TestManagedStopFilterFactory, TestManagedSynonymFilterFactory, TestManagedSynonymGraphFilterFactory, ChangedSchemaMergeTest, CurrencyFieldTypeTest, DateRangeFieldTest, EnumFieldTest, TestBulkSchemaConcurrent, TestUseDocValuesAsStored2, AnalyticsMergeStrategyTest, SortSpecParsingTest, TestCollapseQParserPlugin, TestHashQParserPlugin, TestOverriddenPrefixQueryForCustomFieldType, TestQueryWrapperFilter, TestRecoveryHdfs, TestOrdValues, TestSortByMinMaxFunction, BlockJoinFacetDistribTest, TestCloudNestedDocsSort, TestExactStatsCache, PKIAuthenticationIntegrationTest, TestAnalyzeInfixSuggestions, TestBlendedInfixSuggestions, TestFreeTextSuggestions, BlockCacheTest, BlockDirectoryTest, TestDocTermOrds, CloneFieldUpdateProcessorFactoryTest, TestRTimerTree, TestTestInjection, FullHLLTest, HLLSerializationTest]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=HLLSerializationTest -Dtests.seed=36D99D01EC357CA7 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=zh-SG -Dtests.timezone=America/Kralendijk -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   0.00s J0 | HLLSerializationTest (suite) <<<
   [junit4]    > Throwable #1: java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4] Completed [743/745 (30!)] on J0 in 1.72s, 0 tests, 1 error <<< FAILURES!

[...truncated 1 lines...]
   [junit4] JVM J0: stdout was not empty, see: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J0-20171023_004358_4652127400770976251597.sysout
   [junit4] >>> JVM J0 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4] Dumping heap to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps/java_pid17481.hprof ...
   [junit4] Heap dump file created [465556305 bytes in 15.807 secs]
   [junit4] <<< JVM J0: EOF ----

[...truncated 8481 lines...]
BUILD FAILED
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:834: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:778: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid17481.hprof

Total time: 765 minutes 2 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any

[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1405 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1405/

4 tests failed.
FAILED:  org.apache.lucene.index.TestIndexWriterOnVMError.testCheckpoint

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
	at __randomizedtesting.SeedInfo.seed([3F84C2C7FF31062D]:0)


FAILED:  junit.framework.TestSuite.org.apache.lucene.index.TestIndexWriterOnVMError

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
	at __randomizedtesting.SeedInfo.seed([3F84C2C7FF31062D]:0)


FAILED:  org.apache.solr.cloud.TestTolerantUpdateProcessorRandomCloud.testRandomUpdates

Error Message:
Could not load collection from ZK: test_col

Stack Trace:
org.apache.solr.common.SolrException: Could not load collection from ZK: test_col
	at __randomizedtesting.SeedInfo.seed([A5493FAFE4F0B95B:9988EA1F938D4749]:0)
	at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1172)
	at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:692)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.getDocCollection(CloudSolrClient.java:1188)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:832)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:800)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:957)
	at org.apache.solr.cloud.TestTolerantUpdateProcessorRandomCloud.countDocs(TestTolerantUpdateProcessorRandomCloud.java:367)
	at org.apache.solr.cloud.TestTolerantUpdateProcessorRandomCloud.testRandomUpdates(TestTolerantUpdateProcessorRandomCloud.java:277)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/test_col/state.json
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
	at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212)
	at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:357)
	at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:354)
	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
	at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)
	at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1184)
	at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1170)
	... 48 more


FAILED:  org.apache.solr.cloud.hdfs.HdfsUnloadDistributedZkTest.test

Error Message:
Captured an uncaught exception in thread: Thread[id=3663, name=testExecutor-967-thread-15, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=3663, name=testExecutor-967-thread-15, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest]
Caused by: java.lang.RuntimeException: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:57807
	at __randomizedtesting.SeedInfo.seed([A5493FAFE4F0B95B]:0)
	at org.apache.solr.cloud.BasicDistributedZkTest.lambda$createCollectionInOneInstance$0(BasicDistributedZkTest.java:595)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:57807
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:637)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
	at org.apache.solr.cloud.BasicDistributedZkTest.lambda$createCollectionInOneInstance$0(BasicDistributedZkTest.java:593)
	... 4 more
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:171)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:525)
	... 9 more




Build Log:
[...truncated 982 lines...]
   [junit4] Suite: org.apache.lucene.index.TestIndexWriterOnVMError
   [junit4]   2> d’oct. 22, 2017 2:26:58 AM com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
   [junit4]   2> WARNING: Suite execution timed out: org.apache.lucene.index.TestIndexWriterOnVMError
   [junit4]   2>    1) Thread[id=9, name=JUnit4-serializer-daemon, state=TIMED_WAITING, group=main]
   [junit4]   2>         at java.lang.Thread.sleep(Native Method)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:50)
   [junit4]   2>    2) Thread[id=33, name=SUITE-TestIndexWriterOnVMError-seed#[3F84C2C7FF31062D], state=RUNNABLE, group=TGRP-TestIndexWriterOnVMError]
   [junit4]   2>         at java.lang.Thread.getStackTrace(Thread.java:1559)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:696)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:693)
   [junit4]   2>         at java.security.AccessController.doPrivileged(Native Method)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getStackTrace(ThreadLeakControl.java:693)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getThreadsWithTraces(ThreadLeakControl.java:709)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.formatThreadStacksFull(ThreadLeakControl.java:689)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.access$1000(ThreadLeakControl.java:65)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$2.evaluate(ThreadLeakControl.java:415)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:705)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.access$200(RandomizedRunner.java:139)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$2.run(RandomizedRunner.java:626)
   [junit4]   2>    3) Thread[id=1, name=main, state=WAITING, group=main]
   [junit4]   2>         at java.lang.Object.wait(Native Method)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1252)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1326)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:636)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:493)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:251)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
   [junit4]   2>    4) Thread[id=34, name=TEST-TestIndexWriterOnVMError.testCheckpoint-seed#[3F84C2C7FF31062D], state=RUNNABLE, group=TGRP-TestIndexWriterOnVMError]
   [junit4]   2>         at java.lang.Thread.yield(Native Method)
   [junit4]   2>         at org.apache.lucene.store.MockDirectoryWrapper.maybeYield(MockDirectoryWrapper.java:626)
   [junit4]   2>         at org.apache.lucene.store.MockDirectoryWrapper.openInput(MockDirectoryWrapper.java:745)
   [junit4]   2>         at org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:99)
   [junit4]   2>         at org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:99)
   [junit4]   2>         at org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:99)
   [junit4]   2>         at org.apache.lucene.store.Directory.openChecksumInput(Directory.java:119)
   [junit4]   2>         at org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.write(Lucene50CompoundFormat.java:93)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.createCompoundFile(IndexWriter.java:4957)
   [junit4]   2>         at org.apache.lucene.index.DocumentsWriterPerThread.sealFlushedSegment(DocumentsWriterPerThread.java:538)
   [junit4]   2>         at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:490)
   [junit4]   2>         at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:542)
   [junit4]   2>         at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:658)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:453)
   [junit4]   2>         at org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:103)
   [junit4]   2>         at org.apache.lucene.index.TestIndexWriterOnVMError.doTest(TestIndexWriterOnVMError.java:175)
   [junit4]   2>         at org.apache.lucene.index.TestIndexWriterOnVMError.testCheckpoint(TestIndexWriterOnVMError.java:280)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestIndexWriterOnVMError -Dtests.method=testCheckpoint -Dtests.seed=3F84C2C7FF31062D -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=ca-ES -Dtests.timezone=Pacific/Kwajalein -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   7119s J0 | TestIndexWriterOnVMError.testCheckpoint <<<
   [junit4]    > Throwable #1: java.lang.Exception: Test abandoned because suite timeout was reached.
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([3F84C2C7FF31062D]:0)
   [junit4]   2> d’oct. 22, 2017 2:26:58 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> d’oct. 22, 2017 2:27:20 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 1 thread leaked from SUITE scope at org.apache.lucene.index.TestIndexWriterOnVMError: 
   [junit4]   2>    1) Thread[id=34, name=TEST-TestIndexWriterOnVMError.testCheckpoint-seed#[3F84C2C7FF31062D], state=RUNNABLE, group=TGRP-TestIndexWriterOnVMError]
   [junit4]   2>         at java.lang.Throwable.getStackTraceElement(Native Method)
   [junit4]   2>         at java.lang.Throwable.getOurStackTrace(Throwable.java:827)
   [junit4]   2>         at java.lang.Throwable.getStackTrace(Throwable.java:816)
   [junit4]   2>         at java.lang.Thread.getStackTrace(Thread.java:1559)
   [junit4]   2>         at org.apache.lucene.index.TestIndexWriterOnVMError$4.eval(TestIndexWriterOnVMError.java:283)
   [junit4]   2>         at org.apache.lucene.store.MockDirectoryWrapper.maybeThrowDeterministicException(MockDirectoryWrapper.java:1022)
   [junit4]   2>         at org.apache.lucene.store.MockIndexOutputWrapper.writeBytes(MockIndexOutputWrapper.java:144)
   [junit4]   2>         at org.apache.lucene.store.MockIndexOutputWrapper.writeByte(MockIndexOutputWrapper.java:126)
   [junit4]   2>         at org.apache.lucene.store.DataOutput.writeInt(DataOutput.java:72)
   [junit4]   2>         at org.apache.lucene.codecs.lucene70.Lucene70DocValuesConsumer.addNumericField(Lucene70DocValuesConsumer.java:107)
   [junit4]   2>         at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:109)
   [junit4]   2>         at org.apache.lucene.index.NumericDocValuesWriter.flush(NumericDocValuesWriter.java:117)
   [junit4]   2>         at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:247)
   [junit4]   2>         at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:131)
   [junit4]   2>         at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:451)
   [junit4]   2>         at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:542)
   [junit4]   2>         at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:658)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.prepareCommitInternal(IndexWriter.java:3199)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.commitInternal(IndexWriter.java:3411)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3376)
   [junit4]   2>         at org.apache.lucene.index.TestIndexWriterOnVMError.doTest(TestIndexWriterOnVMError.java:181)
   [junit4]   2>         at org.apache.lucene.index.TestIndexWriterOnVMError.testCheckpoint(TestIndexWriterOnVMError.java:280)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> d’oct. 22, 2017 2:27:20 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=34, name=TEST-TestIndexWriterOnVMError.testCheckpoint-seed#[3F84C2C7FF31062D], state=RUNNABLE, group=TGRP-TestIndexWriterOnVMError]
   [junit4]   2> d’oct. 22, 2017 2:27:24 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> SEVERE: There are still zombie threads that couldn't be terminated:
   [junit4]   2>    1) Thread[id=34, name=TEST-TestIndexWriterOnVMError.testCheckpoint-seed#[3F84C2C7FF31062D], state=RUNNABLE, group=TGRP-TestIndexWriterOnVMError]
   [junit4]   2>         at java.lang.Throwable.getStackTraceElement(Native Method)
   [junit4]   2>         at java.lang.Throwable.getOurStackTrace(Throwable.java:827)
   [junit4]   2>         at java.lang.Throwable.getStackTrace(Throwable.java:816)
   [junit4]   2>         at java.lang.Thread.getStackTrace(Thread.java:1559)
   [junit4]   2>         at org.apache.lucene.index.TestIndexWriterOnVMError$4.eval(TestIndexWriterOnVMError.java:283)
   [junit4]   2>         at org.apache.lucene.store.MockDirectoryWrapper.maybeThrowDeterministicException(MockDirectoryWrapper.java:1022)
   [junit4]   2>         at org.apache.lucene.store.MockDirectoryWrapper.sync(MockDirectoryWrapper.java:196)
   [junit4]   2>         at org.apache.lucene.store.LockValidatingDirectoryWrapper.sync(LockValidatingDirectoryWrapper.java:68)
   [junit4]   2>         at org.apache.lucene.index.SegmentInfos.write(SegmentInfos.java:445)
   [junit4]   2>         at org.apache.lucene.index.SegmentInfos.prepareCommit(SegmentInfos.java:745)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.startCommit(IndexWriter.java:4740)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.prepareCommitInternal(IndexWriter.java:3279)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.commitInternal(IndexWriter.java:3411)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3376)
   [junit4]   2>         at org.apache.lucene.index.TestIndexWriterOnVMError.doTest(TestIndexWriterOnVMError.java:181)
   [junit4]   2>         at org.apache.lucene.index.TestIndexWriterOnVMError.testCheckpoint(TestIndexWriterOnVMError.java:280)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {text_payloads=FSTOrd50, text_vectors=Lucene50(blocksize=128), text1=PostingsFormat(name=LuceneVarGapDocFreqInterval), id=PostingsFormat(name=MockRandom)}, docValues:{dv3=DocValuesFormat(name=Direct), dv2=DocValuesFormat(name=Memory), dv5=DocValuesFormat(name=Asserting), dv=DocValuesFormat(name=Lucene70), text_payloads=DocValuesFormat(name=Direct), dv4=DocValuesFormat(name=Lucene70), text_vectors=DocValuesFormat(name=Lucene70), text1=DocValuesFormat(name=Memory), id=DocValuesFormat(name=Asserting), point2d=DocValuesFormat(name=Lucene70), stored1=DocValuesFormat(name=Lucene70), point=DocValuesFormat(name=Memory)}, maxPointsInLeafNode=1400, maxMBSortInHeap=6.782718381460376, sim=RandomSimilarity(queryNorm=true): {text_payloads=DFR I(n)2, text_vectors=DFR I(ne)LZ(0.3), text1=DFR I(F)L2}, locale=ca-ES, timezone=Pacific/Kwajalein
   [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=2,free=336198488,total=392167424
   [junit4]   2> NOTE: All tests run in this JVM: [Test2BPoints, TestSmallFloat, TestDoubleRangeFieldQueries, TestIndexInput, TestSpanBoostQuery, TestUnicodeUtil, TestNewestSegment, TestSimpleExplanations, TestSearch, TestSort, TestDocIDMerger, TestIndexWriterOnVMError]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestIndexWriterOnVMError -Dtests.seed=3F84C2C7FF31062D -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=ca-ES -Dtests.timezone=Pacific/Kwajalein -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J0 | TestIndexWriterOnVMError (suite) <<<
   [junit4]    > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([3F84C2C7FF31062D]:0)
   [junit4] Completed [131/455 (1!)] on J0 in 7226.34s, 3 tests, 2 errors <<< FAILURES!

[...truncated 11729 lines...]
   [junit4] Suite: org.apache.solr.cloud.hdfs.HdfsUnloadDistributedZkTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_A5493FAFE4F0B95B-001/init-core-data-001
   [junit4]   2> 1354994 WARN  (SUITE-HdfsUnloadDistributedZkTest-seed#[A5493FAFE4F0B95B]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
   [junit4]   2> 1354994 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[A5493FAFE4F0B95B]-worker) [    ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 1355006 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[A5493FAFE4F0B95B]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
   [junit4]   2> 1355006 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[A5493FAFE4F0B95B]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 1355006 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[A5493FAFE4F0B95B]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
   [junit4]   1> Formatting using clusterid: testClusterID
   [junit4]   2> 1355433 WARN  (SUITE-HdfsUnloadDistributedZkTest-seed#[A5493FAFE4F0B95B]-worker) [    ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
   [junit4]   2> 1355534 WARN  (SUITE-HdfsUnloadDistributedZkTest-seed#[A5493FAFE4F0B95B]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 1355536 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[A5493FAFE4F0B95B]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 1355611 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[A5493FAFE4F0B95B]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/hdfs to ./temp/Jetty_localhost_40345_hdfs____615x2a/webapp
   [junit4]   2> 1357636 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[A5493FAFE4F0B95B]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:40345
   [junit4]   2> 1359318 WARN  (SUITE-HdfsUnloadDistributedZkTest-seed#[A5493FAFE4F0B95B]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 1359319 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[A5493FAFE4F0B95B]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 1359380 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[A5493FAFE4F0B95B]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_52455_datanode____.iqgcjm/webapp
   [junit4]   2> 1363995 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[A5493FAFE4F0B95B]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:52455
   [junit4]   2> 1365777 WARN  (SUITE-HdfsUnloadDistributedZkTest-seed#[A5493FAFE4F0B95B]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 1365778 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[A5493FAFE4F0B95B]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 1366016 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[A5493FAFE4F0B95B]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_57854_datanode____.reg338/webapp
   [junit4]   2> 1367414 ERROR (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_A5493FAFE4F0B95B-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_A5493FAFE4F0B95B-001/tempDir-001/hdfsBaseDir/data/data2/]]  heartbeating to localhost/127.0.0.1:54294) [    ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
   [junit4]   2> 1367730 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x1f82a80d2deb12: from storage DS-fca562bf-737b-4371-ba5c-bb130d85e061 node DatanodeRegistration(127.0.0.1:48746, datanodeUuid=7d7160b0-ba17-4756-b07c-6a92ad364696, infoPort=42712, infoSecurePort=0, ipcPort=38430, storageInfo=lv=-56;cid=testClusterID;nsid=39758302;c=0), blocks: 0, hasStaleStorage: true, processing time: 85 msecs
   [junit4]   2> 1367730 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x1f82a80d2deb12: from storage DS-a575ad0c-6fdf-4f8b-8bf5-09266bb4f2f0 node DatanodeRegistration(127.0.0.1:48746, datanodeUuid=7d7160b0-ba17-4756-b07c-6a92ad364696, infoPort=42712, infoSecurePort=0, ipcPort=38430, storageInfo=lv=-56;cid=testClusterID;nsid=39758302;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 1369216 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[A5493FAFE4F0B95B]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:57854
   [junit4]   2> 1371311 ERROR (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_A5493FAFE4F0B95B-001/tempDir-001/hdfsBaseDir/data/data3/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_A5493FAFE4F0B95B-001/tempDir-001/hdfsBaseDir/data/data4/]]  heartbeating to localhost/127.0.0.1:54294) [    ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
   [junit4]   2> 1371522 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x1f82a8f2c9c9f0: from storage DS-1dd0a93b-1acf-4081-9214-4f944e06dab9 node DatanodeRegistration(127.0.0.1:34452, datanodeUuid=db721d53-2483-4484-a64e-59864fae0dc3, infoPort=41667, infoSecurePort=0, ipcPort=56495, storageInfo=lv=-56;cid=testClusterID;nsid=39758302;c=0), blocks: 0, hasStaleStorage: true, processing time: 20 msecs
   [junit4]   2> 1371522 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x1f82a8f2c9c9f0: from storage DS-6b68f25c-d168-4f3b-8f7c-836363f58817 node DatanodeRegistration(127.0.0.1:34452, datanodeUuid=db721d53-2483-4484-a64e-59864fae0dc3, infoPort=41667, infoSecurePort=0, ipcPort=56495, storageInfo=lv=-56;cid=testClusterID;nsid=39758302;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 1372952 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1372958 INFO  (Thread-1267) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1372958 INFO  (Thread-1267) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1372998 ERROR (Thread-1267) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 1373058 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.a.s.c.ZkTestServer start zk server on port:57952
   [junit4]   2> 1373138 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1373139 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1373140 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1373140 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1373141 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1373142 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1373142 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1373143 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1373144 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1373144 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1373145 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1373146 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
   [junit4]   2> 1377516 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1377820 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@788385b{/,null,AVAILABLE}
   [junit4]   2> 1377820 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@4df82296{HTTP/1.1,[http/1.1]}{127.0.0.1:60987}
   [junit4]   2> 1377820 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.e.j.s.Server Started @1396754ms
   [junit4]   2> 1377836 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:54294/hdfs__localhost_54294__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-master_checkout_solr_build_solr-core_test_J2_temp_solr.cloud.hdfs.HdfsUnloadDistributedZkTest_A5493FAFE4F0B95B-001_tempDir-002_control_data, hostContext=/, hostPort=60987, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_A5493FAFE4F0B95B-001/control-001/cores}
   [junit4]   2> 1377839 ERROR (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1377839 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 1377839 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1377840 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 1377840 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-21T20:24:27.520Z
   [junit4]   2> 1378767 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 1378767 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_A5493FAFE4F0B95B-001/control-001/solr.xml
   [junit4]   2> 1378823 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 1378823 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 1378824 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@6f93ec55, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 1378970 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:57952/solr
   [junit4]   2> 1379626 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [n:127.0.0.1:60987_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1379627 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [n:127.0.0.1:60987_    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:60987_
   [junit4]   2> 1379627 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [n:127.0.0.1:60987_    ] o.a.s.c.Overseer Overseer (id=98868754032754692-127.0.0.1:60987_-n_0000000000) starting
   [junit4]   2> 1379952 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [n:127.0.0.1:60987_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60987_
   [junit4]   2> 1380058 INFO  (zkCallback-258-thread-1-processing-n:127.0.0.1:60987_) [n:127.0.0.1:60987_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1382846 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [n:127.0.0.1:60987_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6f93ec55
   [junit4]   2> 1382876 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [n:127.0.0.1:60987_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6f93ec55
   [junit4]   2> 1382876 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [n:127.0.0.1:60987_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6f93ec55
   [junit4]   2> 1382884 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [n:127.0.0.1:60987_    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_A5493FAFE4F0B95B-001/control-001/cores
   [junit4]   2> 1383585 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1383590 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:57952/solr ready
   [junit4]   2> 1383813 INFO  (qtp1953859315-3029) [n:127.0.0.1:60987_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:60987_&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1384159 INFO  (OverseerThreadFactory-888-thread-1-processing-n:127.0.0.1:60987_) [n:127.0.0.1:60987_    ] o.a.s.c.CreateCollectionCmd Create collection control_collection
   [junit4]   2> 1384346 INFO  (qtp1953859315-3027) [n:127.0.0.1:60987_    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1384347 INFO  (qtp1953859315-3027) [n:127.0.0.1:60987_    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 1384488 INFO  (zkCallback-258-thread-1-processing-n:127.0.0.1:60987_) [n:127.0.0.1:60987_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1385673 INFO  (qtp1953859315-3027) [n:127.0.0.1:60987_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 1386025 INFO  (qtp1953859315-3027) [n:127.0.0.1:60987_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
   [junit4]   2> 1396476 WARN  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]-SendThread(127.0.0.1:57952)) [    ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 12886ms for sessionid 0x15f409ba3820005
   [junit4]   2> 1396478 WARN  (org.apache.hadoop.util.JvmPauseMonitor$Monitor@2828bfa4) [    ] o.a.h.u.JvmPauseMonitor Detected pause in JVM or host machine (eg GC): pause of approximately 10090ms
   [junit4]   2> GC pool 'PS MarkSweep' had collection(s): count=1 time=10173ms
   [junit4]   2> GC pool 'PS Scavenge' had collection(s): count=1 time=77ms
   [junit4]   2> 1396498 WARN  (org.apache.hadoop.util.JvmPauseMonitor$Monitor@33086bd8) [    ] o.a.h.u.JvmPauseMonitor Detected pause in JVM or host machine (eg GC): pause of approximately 10111ms
   [junit4]   2> GC pool 'PS MarkSweep' had collection(s): count=1 time=10173ms
   [junit4]   2> GC pool 'PS Scavenge' had collection(s): count=1 time=77ms
   [junit4]   2> 1396498 WARN  (org.apache.hadoop.util.JvmPauseMonitor$Monitor@788367e8) [    ] o.a.h.u.JvmPauseMonitor Detected pause in JVM or host machine (eg GC): pause of approximately 10111ms
   [junit4]   2> GC pool 'PS MarkSweep' had collection(s): count=1 time=10173ms
   [junit4]   2> GC pool 'PS Scavenge' had collection(s): count=1 time=77ms
   [junit4]   2> 1396478 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f409ba3820005, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1398026 WARN  (zkCallback-263-thread-2) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@71048af0 name: ZooKeeperConnection Watcher:127.0.0.1:57952/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
   [junit4]   2> 1398026 WARN  (zkCallback-263-thread-2) [    ] o.a.s.c.c.ConnectionManager zkClient has disconnected
   [junit4]   2> 1398636 WARN  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]-SendThread(127.0.0.1:57952)) [    ] o.a.z.ClientCnxn Unable to reconnect to ZooKeeper service, session 0x15f409ba3820005 has expired
   [junit4]   2> 1398705 WARN  (zkCallback-263-thread-3) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@71048af0 name: ZooKeeperConnection Watcher:127.0.0.1:57952/solr got event WatchedEvent state:Expired type:None path:null path: null type: None
   [junit4]   2> 1398705 WARN  (zkCallback-263-thread-3) [    ] o.a.s.c.c.ConnectionManager Our previous ZooKeeper session was expired. Attempting to reconnect to recover relationship with ZooKeeper...
   [junit4]   2> 1398705 WARN  (zkCallback-263-thread-3) [    ] o.a.s.c.c.DefaultConnectionStrategy Connection expired - starting a new one...
   [junit4]   2> 1398762 INFO  (zkCallback-263-thread-3) [    ] o.a.s.c.c.ConnectionManager Connection with ZooKeeper reestablished.
   [junit4]   2> 1398784 INFO  (zkCallback-263-thread-3) [    ] o.a.s.c.c.DefaultConnectionStrategy Reconnected to ZooKeeper
   [junit4]   2> 1398784 INFO  (zkCallback-263-thread-3) [    ] o.a.s.c.c.ConnectionManager Connected:true
   [junit4]   2> 1398870 INFO  (qtp1953859315-3027) [n:127.0.0.1:60987_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 1399002 INFO  (qtp1953859315-3027) [n:127.0.0.1:60987_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
   [junit4]   2> 1399014 INFO  (qtp1953859315-3027) [n:127.0.0.1:60987_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6f93ec55
   [junit4]   2> 1399014 INFO  (qtp1953859315-3027) [n:127.0.0.1:60987_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:54294/solr_hdfs_home
   [junit4]   2> 1399014 INFO  (qtp1953859315-3027) [n:127.0.0.1:60987_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
   [junit4]   2> 1399014 INFO  (qtp1953859315-3027) [n:127.0.0.1:60987_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1399014 INFO  (qtp1953859315-3027) [n:127.0.0.1:60987_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_A5493FAFE4F0B95B-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[hdfs://localhost:54294/solr_hdfs_home/control_collection/core_node2/data/]
   [junit4]   2> 1399015 INFO  (qtp1953859315-3027) [n:127.0.0.1:60987_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:54294/solr_hdfs_home/control_collection/core_node2/data/snapshot_metadata
   [junit4]   2> 1399131 INFO  (qtp1953859315-3027) [n:127.0.0.1:60987_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
   [junit4]   2> 1399131 INFO  (qtp1953859315-3027) [n:127.0.0.1:60987_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
   [junit4]   2> 1399471 INFO  (qtp1953859315-3027) [n:127.0.0.1:60987_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 1399472 INFO  (qtp1953859315-3027) [n:127.0.0.1:60987_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:54294/solr_hdfs_home/control_collection/core_node2/data
   [junit4]   2> 1399646 INFO  (qtp1953859315-3027) [n:127.0.0.1:60987_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:54294/solr_hdfs_home/control_collection/core_node2/data/index
   [junit4]   2> 1399738 INFO  (qtp1953859315-3027) [n:127.0.0.1:60987_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
   [junit4]   2> 1399738 INFO  (qtp1953859315-3027) [n:127.0.0.1:60987_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
   [junit4]   2> 1399757 INFO  (qtp1953859315-3027) [n:127.0.0.1:60987_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 1399758 INFO  (qtp1953859315-3027) [n:127.0.0.1:60987_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=33, maxMergeAtOnceExplicit=12, maxMergedSegmentMB=57.3876953125, floorSegmentMB=2.068359375, forceMergeDeletesPctAllowed=1.0906791817993011, segmentsPerTier=43.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8261513871282599
   [junit4]   2> 1400326 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:34452 is added to blk_1073741825_1001{UCState=COMMITTED, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-fca562bf-737b-4371-ba5c-bb130d85e061:NORMAL:127.0.0.1:48746|RBW], ReplicaUC[[DISK]DS-1dd0a93b-1acf-4081-9214-4f944e06dab9:NORMAL:127.0.0.1:34452|RBW]]} size 72
   [junit4]   2> 1400326 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:48746 is added to blk_1073741825_1001 size 72
   [junit4]   2> 1400989 WARN  (qtp1953859315-3027) [n:127.0.0.1:60987_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 1401514 INFO  (qtp1953859315-3027) [n:127.0.0.1:60987_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
   [junit4]   2> 1401514 INFO  (qtp1953859315-3027) [n:127.0.0.1:60987_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1401515 INFO  (qtp1953859315-3027) [n:127.0.0.1:60987_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
   [junit4]   2> 1401604 INFO  (qtp1953859315-3027) [n:127.0.0.1:60987_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1401604 INFO  (qtp1953859315-3027) [n:127.0.0.1:60987_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1401674 INFO  (qtp1953859315-3027) [n:127.0.0.1:60987_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=25, maxMergeAtOnceExplicit=31, maxMergedSegmentMB=79.849609375, floorSegmentMB=0.7177734375, forceMergeDeletesPctAllowed=15.49613221303093, segmentsPerTier=45.0, maxCFSSegmentSizeMB=1.49609375, noCFSRatio=0.18809284224528106
   [junit4]   2> 1401948 INFO  (qtp1953859315-3027) [n:127.0.0.1:60987_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@5409c8a6[control_collection_shard1_replica_n1] main]
   [junit4]   2> 1401954 INFO  (qtp1953859315-3027) [n:127.0.0.1:60987_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1401954 INFO  (qtp1953859315-3027) [n:127.0.0.1:60987_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1401955 INFO  (qtp1953859315-3027) [n:127.0.0.1:60987_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 1402000 INFO  (searcherExecutor-891-thread-1-processing-n:127.0.0.1:60987_ x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:60987_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@5409c8a6[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1402026 INFO  (qtp1953859315-3027) [n:127.0.0.1:60987_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1581900094983110656
   [junit4]   2> 1402054 INFO  (qtp1953859315-3027) [n:127.0.0.1:60987_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1402054 INFO  (qtp1953859315-3027) [n:127.0.0.1:60987_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1402054 INFO  (qtp1953859315-3027) [n:127.0.0.1:60987_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:60987/control_collection_shard1_replica_n1/
   [junit4]   2> 1402054 INFO  (qtp1953859315-3027) [n:127.0.0.1:60987_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 1402054 INFO  (qtp1953859315-3027) [n:127.0.0.1:60987_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:60987/control_collection_shard1_replica_n1/ has no replicas
   [junit4]   2> 1402055 INFO  (qtp1953859315-3027) [n:127.0.0.1:60987_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 1402076 INFO  (qtp1953859315-3027) [n:127.0.0.1:60987_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:60987/control_collection_shard1_replica_n1/ shard1
   [junit4]   2> 1402210 INFO  (zkCallback-258-thread-2-processing-n:127.0.0.1:60987_) [n:127.0.0.1:60987_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1402211 INFO  (qtp1953859315-3027) [n:127.0.0.1:60987_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1402230 INFO  (qtp1953859315-3027) [n:127.0.0.1:60987_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=17884
   [junit4]   2> 1402249 INFO  (qtp1953859315-3029) [n:127.0.0.1:60987_    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 1402334 INFO  (zkCallback-258-thread-2-processing-n:127.0.0.1:60987_) [n:127.0.0.1:60987_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1403484 INFO  (qtp1953859315-3029) [n:127.0.0.1:60987_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:60987_&wt=javabin&version=2} status=0 QTime=19670
   [junit4]   2> 1403554 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1403555 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:57952/solr ready
   [junit4]   2> 1403555 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1403556 INFO  (qtp1953859315-3033) [n:127.0.0.1:60987_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1403574 INFO  (OverseerThreadFactory-888-thread-2-processing-n:127.0.0.1:60987_) [n:127.0.0.1:60987_    ] o.a.s.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 1403594 INFO  (OverseerCollectionConfigSetProcessor-98868754032754692-127.0.0.1:60987_-n_0000000000) [n:127.0.0.1:60987_    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1403605 WARN  (OverseerThreadFactory-888-thread-2-processing-n:127.0.0.1:60987_) [n:127.0.0.1:60987_    ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
   [junit4]   2> 1403894 INFO  (qtp1953859315-3033) [n:127.0.0.1:60987_    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 1403894 INFO  (qtp1953859315-3033) [n:127.0.0.1:60987_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=337
   [junit4]   2> 1403914 INFO  (OverseerCollectionConfigSetProcessor-98868754032754692-127.0.0.1:60987_-n_0000000000) [n:127.0.0.1:60987_    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1405088 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_A5493FAFE4F0B95B-001/shard-1-001 of type NRT
   [junit4]   2> 1405088 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1405132 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@62c5cbf9{/,null,AVAILABLE}
   [junit4]   2> 1405132 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@d7e5de4{HTTP/1.1,[http/1.1]}{127.0.0.1:57807}
   [junit4]   2> 1405132 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.e.j.s.Server Started @1424066ms
   [junit4]   2> 1405132 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:54294/hdfs__localhost_54294__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-master_checkout_solr_build_solr-core_test_J2_temp_solr.cloud.hdfs.HdfsUnloadDistributedZkTest_A5493FAFE4F0B95B-001_tempDir-002_jetty1, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/, hostPort=57807, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_A5493FAFE4F0B95B-001/shard-1-001/cores}
   [junit4]   2> 1405133 ERROR (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1405133 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 1405133 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1405133 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 1405133 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-21T20:24:54.813Z
   [junit4]   2> 1405442 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 1405443 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_A5493FAFE4F0B95B-001/shard-1-001/solr.xml
   [junit4]   2> 1405506 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 1405506 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 1405543 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@6f93ec55, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 1405704 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:57952/solr
   [junit4]   2> 1407234 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [n:127.0.0.1:57807_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1407255 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [n:127.0.0.1:57807_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1407276 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [n:127.0.0.1:57807_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57807_
   [junit4]   2> 1407286 INFO  (zkCallback-266-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1407318 INFO  (zkCallback-258-thread-2-processing-n:127.0.0.1:60987_) [n:127.0.0.1:60987_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1407554 INFO  (zkCallback-271-thread-1-processing-n:127.0.0.1:57807_) [n:127.0.0.1:57807_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1411175 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [n:127.0.0.1:57807_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6f93ec55
   [junit4]   2> 1411335 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [n:127.0.0.1:57807_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6f93ec55
   [junit4]   2> 1411335 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [n:127.0.0.1:57807_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6f93ec55
   [junit4]   2> 1411338 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A5493FAFE4F0B95B]) [n:127.0.0.1:57807_    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_A5493FAFE4F0B95B-001/shard-1-001/cores
   [junit4]   2> 1411762 INFO  (qtp908447240-3093) [n:127.0.0.1:57807_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:57807_&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1411935 INFO  (OverseerThreadFactory-888-thread-3-processing-n:127.0.0.1:60987_) [n:127.0.0.1:60987_    ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:57807_ for creating new replica
   [junit4]   2> 1411939 INFO  (qtp908447240-3091) [n:127.0.0.1:57807_    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1411994 INFO  (qtp908447240-3091) [n:127.0.0.1:57807_    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 1412230 INFO  (qtp908447240-3091) [n:127.0.0.1:57807_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 1412394 INFO  (qtp908447240-3091) [n:127.0.0.1:57807_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.s.IndexSchema [collection1_shard2_replica_n41] Schema name=test
   [junit4]   2> 1413670 INFO  (qtp908447240-3091) [n:127.0.0.1:57807_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 1413830 INFO  (qtp908447240-3091) [n:127.0.0.1:57807_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_n41' using configuration from collection collection1, trusted=true
   [junit4]   2> 1413831 INFO  (qtp908447240-3091) [n:127.0.0.1:57807_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_n41' (registry 'solr.core.collection1.shard2.replica_n41') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6f93ec55
   [junit4]   2> 1413831 INFO  (qtp908447240-3091) [n:127.0.0.1:57807_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:54294/solr_hdfs_home
   [junit4]   2> 1413831 INFO  (qtp908447240-3091) [n:127.0.0.1:57807_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
   [junit4]   2> 1413831 INFO  (qtp908447240-3091) [n:127.0.0.1:57807_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1413831 INFO  (qtp908447240-3091) [n:127.0.0.1:57807_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.SolrCore [[collection1_shard2_replica_n41] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_A5493FAFE4F0B95B-001/shard-1-001/cores/collection1_shard2_replica_n41], dataDir=[hdfs://localhost:54294/solr_hdfs_home/collection1/core_node42/data/]
   [junit4]   2> 1413832 INFO  (qtp908447240-3091) [n:127.0.0.1:57807_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:54294/solr_hdfs_home/collection1/core_node42/data/snapshot_metadata
   [junit4]   2> 1414046 INFO  (qtp908447240-3091) [n:127.0.0.1:57807_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
   [junit4]   2> 1414046 INFO  (qtp908447240-3091) [n:127.0.0.1:57807_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
   [junit4]   2> 1414211 INFO  (qtp908447240-3091) [n:127.0.0.1:57807_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 1414229 INFO  (qtp908447240-3091) [n:127.0.0.1:57807_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:54294/solr_hdfs_home/collection1/core_node42/data
   [junit4]   2> 1414386 INFO  (qtp908447240-3091) [n:127.0.0.1:57807_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:54294/solr_hdfs_home/collection1/core_node42/data/index
   [junit4]   2> 1414483 INFO  (qtp908447240-3091) [n:127.0.0.1:57807_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
   [junit4]   2> 1414483 INFO  (qtp908447240-3091) [n:127.0.0.1:57807_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
   [junit4]   2> 1414578 INFO  (qtp908447240-3091) [n:127.0.0.1:57807_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 1414579 INFO  (qtp908447240-3091) [n:127.0.0.1:57807_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=33, maxMergeAtOnceExplicit=12, maxMergedSegmentMB=57.3876953125, floorSegmentMB=2.068359375, forceMergeDeletesPctAllowed=1.0906791817993011, segmentsPerTier=43.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8261513871282599
   [junit4]   2> 1415230 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:48746 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-1dd0a93b-1acf-4081-9214-4f944e06dab9:NORMAL:127.0.0.1:34452|RBW], ReplicaUC[[DISK]DS-fca562bf-737b-4371-ba5c-bb130d85e061:NORMAL:127.0.0.1:48746|RBW]]} size 0
   [junit4]   2> 1415231 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:34452 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-1dd0a93b-1acf-4081-9214-4f944e06dab9:NORMAL:127.0.0.1:34452|RBW], ReplicaUC[[DISK]DS-fca562bf-737b-4371-ba5c-bb130d85e061:NORMAL:127.0.0.1:48746|RBW]]} size 0
   [junit4]   2> 1415233 WARN  (DataStreamer for file /solr_hdfs_home/collection1/core_node42/data/index/pending_segments_1) [n:127.0.0.1:57807_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.h.h.DFSClient Caught exception 
   [junit4]   2> java.lang.InterruptedException
   [junit4]   2> 	at java.lang.Object.wait(Native Method)
   [junit4]   2> 	at java.lang.Thread.join(Thread.java:1252)
   [junit4]   2> 	at java.lang.Thread.join(Thread.java:1326)
   [junit4]   2> 	at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.closeResponder(DFSOutputStream.java:716)
   [junit4]   2> 	at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.endBlock(DFSOutputStream.java:476)
   [junit4]   2> 	at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:652)
   [junit4]   2> 1415241 WARN  (qtp908447240-3091) [n:127.0.0.1:57807_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 1417006 INFO  (qtp908447240-3091) [n:127.0.0.1:57807_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
   [junit4]   2> 1417006 INFO  (qtp908447240-3091) [n:127.0.0.1:57807_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1417006 INFO  (qtp908447240-3091) [n:127.0.0.1:57807_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
   [junit4]   2> 1417110 INFO  (qtp908447240-3091) [n:127.0.0.1:57807_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1417110 INFO  (qtp908447240-3091) [n:127.0.0.1:57807_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1417150 INFO  (qtp908447240-3091) [n:127.0.0.1:57807_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apa

[...truncated too long message...]

kout/solr/core/test-lib/apacheds-interceptors-journal-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-normalization-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-operational-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-referral-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-schema-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-subtree-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-trigger-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-jdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-jdbm1-2.0.0-M2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-kerberos-codec-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-ldif-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-mavibot-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-protocol-kerberos-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-protocol-ldap-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-protocol-shared-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-xdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/api-all-1.0.0-M20.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/bcprov-jdk15-1.45.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/byte-buddy-1.6.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/commons-collections-3.2.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/commons-math3-3.6.1.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/ehcache-core-2.4.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/hadoop-common-2.7.4-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/hadoop-hdfs-2.7.4-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/hadoop-minikdc-2.7.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/htrace-core-3.2.0-incubating.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jersey-core-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jersey-server-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jetty-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jetty-sslengine-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jetty-util-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/mina-core-2.0.0-M5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/mockito-core-2.6.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/netty-all-4.0.36.Final.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/objenesis-2.5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/analysis/icu/lucene-analyzers-icu-8.0.0-SNAPSHOT.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/contrib/solr-analysis-extras/classes/java:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/queryparser/classes/test:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/backward-codecs/classes/test:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/contrib/analysis-extras/lib/icu4j-59.1.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-launcher.jar:/x1/jenkins/.ant/lib/ivy-2.3.0.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-log4j.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit4.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jai.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-javamail.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bsf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-net.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-antlr.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jsch.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-oro.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-logging.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-netrexx.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-testutil.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jdepend.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bcel.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-xalan2.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-resolver.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jmf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-regexp.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-swing.jar:/usr/local/asfpackages/java/jdk1.8.0_144/lib/tools.jar:/x1/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.5.3.jar com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J2-20171021_200109_3997016940757855441139.events @/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J2-20171021_200109_3993528780334380072006.suites -stdin
   [junit4] ERROR: JVM J2 ended with an exception: Forked process returned with error code: 1. Very likely a JVM crash.  See process stdout at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J2-20171021_200109_3991074841708227188313.sysout See process stderr at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J2-20171021_200109_3997161599737057936424.syserr
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1519)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:126)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:982)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:979)
   [junit4] 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4] 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4] 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4] 	at java.lang.Thread.run(Thread.java:748)

BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:834: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:770: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:59: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build.xml:262: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/common-build.xml:556: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/common-build.xml:1494: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/common-build.xml:1021: At least one slave process threw an exception, first: Forked process returned with error code: 1. Very likely a JVM crash.  See process stdout at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J0-20171021_200109_3981390791147888094124.sysout See process stderr at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J0-20171021_200109_398337901991601689147.syserr

Total time: 606 minutes 17 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[Fast Archiver] No artifacts from Lucene-Solr-NightlyTests-master #1402 to compare, so performing full copy of artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any

[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1404 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1404/

5 tests failed.
FAILED:  org.apache.lucene.search.TestInetAddressRangeQueries.testRandomBig

Error Message:
GC overhead limit exceeded

Stack Trace:
java.lang.OutOfMemoryError: GC overhead limit exceeded
	at __randomizedtesting.SeedInfo.seed([9317F840A8061A53:144085CF395F66D3]:0)
	at org.apache.lucene.codecs.memory.DirectPostingsFormat$DirectField$IntArrayWriter.get(DirectPostingsFormat.java:295)
	at org.apache.lucene.codecs.memory.DirectPostingsFormat$DirectField.<init>(DirectPostingsFormat.java:413)
	at org.apache.lucene.codecs.memory.DirectPostingsFormat$DirectFields.<init>(DirectPostingsFormat.java:129)
	at org.apache.lucene.codecs.memory.DirectPostingsFormat.fieldsProducer(DirectPostingsFormat.java:113)
	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.<init>(PerFieldPostingsFormat.java:292)
	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat.fieldsProducer(PerFieldPostingsFormat.java:372)
	at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:112)
	at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:78)
	at org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:208)
	at org.apache.lucene.index.BufferedUpdatesStream$SegmentState.<init>(BufferedUpdatesStream.java:281)
	at org.apache.lucene.index.BufferedUpdatesStream.openSegmentStates(BufferedUpdatesStream.java:309)
	at org.apache.lucene.index.FrozenBufferedUpdates.apply(FrozenBufferedUpdates.java:297)
	at org.apache.lucene.index.DocumentsWriter$ResolveUpdatesEvent.process(DocumentsWriter.java:723)
	at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5055)
	at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5043)
	at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1718)
	at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1460)
	at org.apache.lucene.search.BaseRangeFieldQueryTestCase.verify(BaseRangeFieldQueryTestCase.java:190)
	at org.apache.lucene.search.BaseRangeFieldQueryTestCase.doTestRandom(BaseRangeFieldQueryTestCase.java:160)
	at org.apache.lucene.search.BaseRangeFieldQueryTestCase.testRandomBig(BaseRangeFieldQueryTestCase.java:75)
	at org.apache.lucene.search.TestInetAddressRangeQueries.testRandomBig(TestInetAddressRangeQueries.java:81)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)


FAILED:  org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.testNodeMarkersRegistration

Error Message:
Path /autoscaling/nodeAdded/127.0.0.1:42044_solr wasn't created

Stack Trace:
java.lang.AssertionError: Path /autoscaling/nodeAdded/127.0.0.1:42044_solr wasn't created
	at __randomizedtesting.SeedInfo.seed([CB030D4DAD368C76:D3B98541A3034199]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.testNodeMarkersRegistration(TriggerIntegrationTest.java:909)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.hdfs.HdfsSyncSliceTest.test

Error Message:
Error from server at https://127.0.0.1:40898//control_collection: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 404 </title> </head> <body> <h2>HTTP ERROR: 404</h2> <p>Problem accessing /control_collection/update. Reason: <pre>    Can not find: /control_collection/update</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.20.v20170531</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:40898//control_collection: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 404 </title>
</head>
<body>
<h2>HTTP ERROR: 404</h2>
<p>Problem accessing /control_collection/update. Reason:
<pre>    Can not find: /control_collection/update</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.20.v20170531</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([CB030D4DAD368C76:4357329703CAE18E]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:590)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:887)
	at org.apache.solr.cloud.SyncSliceTest.test(SyncSliceTest.java:73)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.ShardSplitTest.testSplitAfterFailedSplit

Error Message:
expected:<1> but was:<2>

Stack Trace:
java.lang.AssertionError: expected:<1> but was:<2>
	at __randomizedtesting.SeedInfo.seed([CB030D4DAD368C76:324E9EE29143C1FC]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.cloud.ShardSplitTest.testSplitAfterFailedSplit(ShardSplitTest.java:279)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test

Error Message:
Expected numSlices=5 numReplicas=1 but found DocCollection(solrj_collection3//collections/solrj_collection3/state.json/30)={   "pullReplicas":"0",   "replicationFactor":"1",   "shards":{     "shard1":{       "range":"80000000-b332ffff",       "state":"active",       "replicas":{"core_node2":{           "core":"solrj_collection3_shard1_replica_n1",           "base_url":"http://127.0.0.1:43473/qn",           "node_name":"127.0.0.1:43473_qn",           "state":"down",           "type":"NRT"}}},     "shard2":{       "range":"b3330000-e665ffff",       "state":"active",       "replicas":{"core_node5":{           "dataDir":"hdfs://lucene2-us-west.apache.org:43257/solr_hdfs_home/solrj_collection3/core_node5/data/",           "base_url":"http://127.0.0.1:43473/qn",           "node_name":"127.0.0.1:43473_qn",           "type":"NRT",           "ulogDir":"hdfs://lucene2-us-west.apache.org:43257/solr_hdfs_home/solrj_collection3/core_node5/data/tlog",           "core":"solrj_collection3_shard2_replica_n3",           "state":"active",           "leader":"true"}}},     "shard3":{       "range":"e6660000-1998ffff",       "state":"active",       "replicas":{"core_node7":{           "dataDir":"hdfs://lucene2-us-west.apache.org:43257/solr_hdfs_home/solrj_collection3/core_node7/data/",           "base_url":"http://127.0.0.1:43473/qn",           "node_name":"127.0.0.1:43473_qn",           "type":"NRT",           "ulogDir":"hdfs://lucene2-us-west.apache.org:43257/solr_hdfs_home/solrj_collection3/core_node7/data/tlog",           "core":"solrj_collection3_shard3_replica_n4",           "state":"active",           "leader":"true"}}},     "shard4":{       "range":"19990000-4ccbffff",       "state":"active",       "replicas":{"core_node9":{           "dataDir":"hdfs://lucene2-us-west.apache.org:43257/solr_hdfs_home/solrj_collection3/core_node9/data/",           "base_url":"http://127.0.0.1:45531/qn",           "node_name":"127.0.0.1:45531_qn",           "type":"NRT",           "ulogDir":"hdfs://lucene2-us-west.apache.org:43257/solr_hdfs_home/solrj_collection3/core_node9/data/tlog",           "core":"solrj_collection3_shard4_replica_n6",           "state":"active",           "leader":"true"}}},     "shard5":{       "range":"4ccc0000-7fffffff",       "state":"active",       "replicas":{"core_node10":{           "dataDir":"hdfs://lucene2-us-west.apache.org:43257/solr_hdfs_home/solrj_collection3/core_node10/data/",           "base_url":"http://127.0.0.1:43473/qn",           "node_name":"127.0.0.1:43473_qn",           "type":"NRT",           "ulogDir":"hdfs://lucene2-us-west.apache.org:43257/solr_hdfs_home/solrj_collection3/core_node10/data/tlog",           "core":"solrj_collection3_shard5_replica_n8",           "state":"active",           "leader":"true"}}}},   "router":{     "field":"myOwnField",     "name":"compositeId"},   "maxShardsPerNode":"1",   "autoAddReplicas":"true",   "nrtReplicas":"1",   "tlogReplicas":"0"} with /live_nodes: [127.0.0.1:41284_qn, 127.0.0.1:42278_qn, 127.0.0.1:43473_qn, 127.0.0.1:45531_qn, 127.0.0.1:46600_qn]

Stack Trace:
java.lang.AssertionError: Expected numSlices=5 numReplicas=1 but found DocCollection(solrj_collection3//collections/solrj_collection3/state.json/30)={
  "pullReplicas":"0",
  "replicationFactor":"1",
  "shards":{
    "shard1":{
      "range":"80000000-b332ffff",
      "state":"active",
      "replicas":{"core_node2":{
          "core":"solrj_collection3_shard1_replica_n1",
          "base_url":"http://127.0.0.1:43473/qn",
          "node_name":"127.0.0.1:43473_qn",
          "state":"down",
          "type":"NRT"}}},
    "shard2":{
      "range":"b3330000-e665ffff",
      "state":"active",
      "replicas":{"core_node5":{
          "dataDir":"hdfs://lucene2-us-west.apache.org:43257/solr_hdfs_home/solrj_collection3/core_node5/data/",
          "base_url":"http://127.0.0.1:43473/qn",
          "node_name":"127.0.0.1:43473_qn",
          "type":"NRT",
          "ulogDir":"hdfs://lucene2-us-west.apache.org:43257/solr_hdfs_home/solrj_collection3/core_node5/data/tlog",
          "core":"solrj_collection3_shard2_replica_n3",
          "state":"active",
          "leader":"true"}}},
    "shard3":{
      "range":"e6660000-1998ffff",
      "state":"active",
      "replicas":{"core_node7":{
          "dataDir":"hdfs://lucene2-us-west.apache.org:43257/solr_hdfs_home/solrj_collection3/core_node7/data/",
          "base_url":"http://127.0.0.1:43473/qn",
          "node_name":"127.0.0.1:43473_qn",
          "type":"NRT",
          "ulogDir":"hdfs://lucene2-us-west.apache.org:43257/solr_hdfs_home/solrj_collection3/core_node7/data/tlog",
          "core":"solrj_collection3_shard3_replica_n4",
          "state":"active",
          "leader":"true"}}},
    "shard4":{
      "range":"19990000-4ccbffff",
      "state":"active",
      "replicas":{"core_node9":{
          "dataDir":"hdfs://lucene2-us-west.apache.org:43257/solr_hdfs_home/solrj_collection3/core_node9/data/",
          "base_url":"http://127.0.0.1:45531/qn",
          "node_name":"127.0.0.1:45531_qn",
          "type":"NRT",
          "ulogDir":"hdfs://lucene2-us-west.apache.org:43257/solr_hdfs_home/solrj_collection3/core_node9/data/tlog",
          "core":"solrj_collection3_shard4_replica_n6",
          "state":"active",
          "leader":"true"}}},
    "shard5":{
      "range":"4ccc0000-7fffffff",
      "state":"active",
      "replicas":{"core_node10":{
          "dataDir":"hdfs://lucene2-us-west.apache.org:43257/solr_hdfs_home/solrj_collection3/core_node10/data/",
          "base_url":"http://127.0.0.1:43473/qn",
          "node_name":"127.0.0.1:43473_qn",
          "type":"NRT",
          "ulogDir":"hdfs://lucene2-us-west.apache.org:43257/solr_hdfs_home/solrj_collection3/core_node10/data/tlog",
          "core":"solrj_collection3_shard5_replica_n8",
          "state":"active",
          "leader":"true"}}}},
  "router":{
    "field":"myOwnField",
    "name":"compositeId"},
  "maxShardsPerNode":"1",
  "autoAddReplicas":"true",
  "nrtReplicas":"1",
  "tlogReplicas":"0"} with /live_nodes: [127.0.0.1:41284_qn, 127.0.0.1:42278_qn, 127.0.0.1:43473_qn, 127.0.0.1:45531_qn, 127.0.0.1:46600_qn]
	at __randomizedtesting.SeedInfo.seed([CB030D4DAD368C76:4357329703CAE18E]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.assertSliceAndReplicaCount(SharedFSAutoReplicaFailoverTest.java:413)
	at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:255)
	at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:143)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 8974 lines...]
   [junit4] Suite: org.apache.lucene.search.TestInetAddressRangeQueries
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestInetAddressRangeQueries -Dtests.method=testRandomBig -Dtests.seed=9317F840A8061A53 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=en-AU -Dtests.timezone=Asia/Macau -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   1674s J2 | TestInetAddressRangeQueries.testRandomBig <<<
   [junit4]    > Throwable #1: java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([9317F840A8061A53:144085CF395F66D3]:0)
   [junit4]    > 	at org.apache.lucene.codecs.memory.DirectPostingsFormat$DirectField$IntArrayWriter.get(DirectPostingsFormat.java:295)
   [junit4]    > 	at org.apache.lucene.codecs.memory.DirectPostingsFormat$DirectField.<init>(DirectPostingsFormat.java:413)
   [junit4]    > 	at org.apache.lucene.codecs.memory.DirectPostingsFormat$DirectFields.<init>(DirectPostingsFormat.java:129)
   [junit4]    > 	at org.apache.lucene.codecs.memory.DirectPostingsFormat.fieldsProducer(DirectPostingsFormat.java:113)
   [junit4]    > 	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.<init>(PerFieldPostingsFormat.java:292)
   [junit4]    > 	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat.fieldsProducer(PerFieldPostingsFormat.java:372)
   [junit4]    > 	at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:112)
   [junit4]    > 	at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:78)
   [junit4]    > 	at org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:208)
   [junit4]    > 	at org.apache.lucene.index.BufferedUpdatesStream$SegmentState.<init>(BufferedUpdatesStream.java:281)
   [junit4]    > 	at org.apache.lucene.index.BufferedUpdatesStream.openSegmentStates(BufferedUpdatesStream.java:309)
   [junit4]    > 	at org.apache.lucene.index.FrozenBufferedUpdates.apply(FrozenBufferedUpdates.java:297)
   [junit4]    > 	at org.apache.lucene.index.DocumentsWriter$ResolveUpdatesEvent.process(DocumentsWriter.java:723)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5055)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5043)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1718)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1460)
   [junit4]    > 	at org.apache.lucene.search.BaseRangeFieldQueryTestCase.verify(BaseRangeFieldQueryTestCase.java:190)
   [junit4]    > 	at org.apache.lucene.search.BaseRangeFieldQueryTestCase.doTestRandom(BaseRangeFieldQueryTestCase.java:160)
   [junit4]    > 	at org.apache.lucene.search.BaseRangeFieldQueryTestCase.testRandomBig(BaseRangeFieldQueryTestCase.java:75)
   [junit4]    > 	at org.apache.lucene.search.TestInetAddressRangeQueries.testRandomBig(TestInetAddressRangeQueries.java:81)
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/misc/test/J2/temp/lucene.search.TestInetAddressRangeQueries_9317F840A8061A53-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {id=PostingsFormat(name=Direct)}, docValues:{ipRangeField=DocValuesFormat(name=Asserting), id=DocValuesFormat(name=Memory)}, maxPointsInLeafNode=1011, maxMBSortInHeap=7.543160968710648, sim=RandomSimilarity(queryNorm=true): {}, locale=en-AU, timezone=Asia/Macau
   [junit4]   2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=59488640,total=477626368
   [junit4]   2> NOTE: All tests run in this JVM: [TestDiversifiedTopDocsCollector, TestMultiPassIndexSplitter, TestIndexSplitter, TestPKIndexSplitter, TestInetAddressPoint, TestDocValuesStatsCollector, TestInetAddressRangeQueries]
   [junit4] Completed [13/13 (1!)] on J2 in 1772.61s, 4 tests, 1 error <<< FAILURES!

[...truncated 1 lines...]
   [junit4] JVM J2: stdout was not empty, see: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/misc/test/temp/junit4-J2-20171019_210312_948688910555027125359.sysout
   [junit4] >>> JVM J2 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4] Dumping heap to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps/java_pid17465.hprof ...
   [junit4] Heap dump file created [595469265 bytes in 11.494 secs]
   [junit4] <<< JVM J2: EOF ----

[...truncated 4097 lines...]
   [junit4] Suite: org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest
   [junit4]   2> 315349 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[CB030D4DAD368C76]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_CB030D4DAD368C76-001/init-core-data-001
   [junit4]   2> 315350 WARN  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[CB030D4DAD368C76]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=26 numCloses=26
   [junit4]   2> 315350 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[CB030D4DAD368C76]-worker) [    ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 315352 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[CB030D4DAD368C76]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=None)
   [junit4]   2> 315352 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[CB030D4DAD368C76]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /qn/
   [junit4]   1> Formatting using clusterid: testClusterID
   [junit4]   2> 321255 WARN  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[CB030D4DAD368C76]-worker) [    ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
   [junit4]   2> 322052 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[CB030D4DAD368C76]-worker) [    ] o.m.log Logging to org.slf4j.impl.Log4jLoggerAdapter(org.mortbay.log) via org.mortbay.log.Slf4jLog
   [junit4]   2> 322125 WARN  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[CB030D4DAD368C76]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 322477 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[CB030D4DAD368C76]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 322977 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[CB030D4DAD368C76]-worker) [    ] o.m.log Extract jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/hdfs to ./temp/Jetty_lucene2.us.west_apache_org_36955_hdfs____q5yt34/webapp
   [junit4]   2> 325236 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[CB030D4DAD368C76]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@lucene2-us-west.apache.org:36955
   [junit4]   2> 329933 WARN  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[CB030D4DAD368C76]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 329937 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[CB030D4DAD368C76]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 330081 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[CB030D4DAD368C76]-worker) [    ] o.m.log Extract jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_43498_datanode____.qj2dgl/webapp
   [junit4]   2> 332038 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[CB030D4DAD368C76]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:43498
   [junit4]   2> 335789 WARN  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[CB030D4DAD368C76]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 335790 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[CB030D4DAD368C76]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 335994 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[CB030D4DAD368C76]-worker) [    ] o.m.log Extract jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_37154_datanode____.xlgmzx/webapp
   [junit4]   2> 338991 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[CB030D4DAD368C76]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:37154
   [junit4]   2> 343979 ERROR (DataNode: [[[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_CB030D4DAD368C76-001/tempDir-001/hdfsBaseDir/data/data3/, [DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_CB030D4DAD368C76-001/tempDir-001/hdfsBaseDir/data/data4/]]  heartbeating to lucene2-us-west.apache.org/127.0.0.1:43257) [    ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
   [junit4]   2> 344236 ERROR (DataNode: [[[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_CB030D4DAD368C76-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_CB030D4DAD368C76-001/tempDir-001/hdfsBaseDir/data/data2/]]  heartbeating to lucene2-us-west.apache.org/127.0.0.1:43257) [    ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
   [junit4]   2> 346292 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x1e728145f72046: from storage DS-28837cd2-6ce1-49a1-a162-239e582d448b node DatanodeRegistration(127.0.0.1:41645, datanodeUuid=29df2e86-575d-4c35-b894-e9dae72b185f, infoPort=36674, infoSecurePort=0, ipcPort=40049, storageInfo=lv=-56;cid=testClusterID;nsid=1307832034;c=0), blocks: 0, hasStaleStorage: true, processing time: 99 msecs
   [junit4]   2> 346293 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x1e72814726b2d5: from storage DS-beeef937-622b-4a78-b64a-35665c55855e node DatanodeRegistration(127.0.0.1:45153, datanodeUuid=0323785e-ac5d-40e3-acc2-3a5c8ede14e3, infoPort=37479, infoSecurePort=0, ipcPort=38429, storageInfo=lv=-56;cid=testClusterID;nsid=1307832034;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
   [junit4]   2> 346293 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x1e72814726b2d5: from storage DS-ead23820-f5e4-46f7-881c-faf146cf42e4 node DatanodeRegistration(127.0.0.1:45153, datanodeUuid=0323785e-ac5d-40e3-acc2-3a5c8ede14e3, infoPort=37479, infoSecurePort=0, ipcPort=38429, storageInfo=lv=-56;cid=testClusterID;nsid=1307832034;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 346309 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x1e728145f72046: from storage DS-1f6d0eca-5556-4c49-8981-9b1d2abe6077 node DatanodeRegistration(127.0.0.1:41645, datanodeUuid=29df2e86-575d-4c35-b894-e9dae72b185f, infoPort=36674, infoSecurePort=0, ipcPort=40049, storageInfo=lv=-56;cid=testClusterID;nsid=1307832034;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 348518 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 348583 INFO  (Thread-368) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 348583 INFO  (Thread-368) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 348677 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.a.s.c.ZkTestServer start zk server on port:36275
   [junit4]   2> 348714 ERROR (Thread-368) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 348992 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 348993 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 348994 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 349007 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 349008 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 349009 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 349010 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 349027 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 349028 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 349030 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 349100 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 349101 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
   [junit4]   2> 350327 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 350482 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@ca996e1{/qn,null,AVAILABLE}
   [junit4]   2> 350483 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@4730e6d4{HTTP/1.1,[http/1.1]}{127.0.0.1:45531}
   [junit4]   2> 350483 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.e.j.s.Server Started @365846ms
   [junit4]   2> 350483 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {replicaType=NRT, hostContext=/qn, hostPort=45531, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_CB030D4DAD368C76-001/control-001/cores}
   [junit4]   2> 350483 ERROR (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 350483 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 350483 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 350483 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 350483 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-20T00:54:04.003Z
   [junit4]   2> 350610 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 350610 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_CB030D4DAD368C76-001/control-001/solr.xml
   [junit4]   2> 350663 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 350663 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 350664 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@76402bfe, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 350735 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36275/solr
   [junit4]   2> 351451 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [n:127.0.0.1:45531_qn    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 351452 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [n:127.0.0.1:45531_qn    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:45531_qn
   [junit4]   2> 351452 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [n:127.0.0.1:45531_qn    ] o.a.s.c.Overseer Overseer (id=98858489750487044-127.0.0.1:45531_qn-n_0000000000) starting
   [junit4]   2> 351524 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [n:127.0.0.1:45531_qn    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45531_qn
   [junit4]   2> 351524 DEBUG (OverseerAutoScalingTriggerThread-98858489750487044-127.0.0.1:45531_qn-n_0000000000) [n:127.0.0.1:45531_qn    ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 351525 INFO  (zkCallback-406-thread-1-processing-n:127.0.0.1:45531_qn) [n:127.0.0.1:45531_qn    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 351560 DEBUG (OverseerAutoScalingTriggerThread-98858489750487044-127.0.0.1:45531_qn-n_0000000000) [n:127.0.0.1:45531_qn    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
   [junit4]   2> 351561 DEBUG (OverseerAutoScalingTriggerThread-98858489750487044-127.0.0.1:45531_qn-n_0000000000) [n:127.0.0.1:45531_qn    ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:45531_qn]
   [junit4]   2> 351597 DEBUG (OverseerAutoScalingTriggerThread-98858489750487044-127.0.0.1:45531_qn-n_0000000000) [n:127.0.0.1:45531_qn    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion -1
   [junit4]   2> 351597 DEBUG (OverseerAutoScalingTriggerThread-98858489750487044-127.0.0.1:45531_qn-n_0000000000) [n:127.0.0.1:45531_qn    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
   [junit4]   2> 351650 DEBUG (OverseerAutoScalingTriggerThread-98858489750487044-127.0.0.1:45531_qn-n_0000000000) [n:127.0.0.1:45531_qn    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 351650 DEBUG (OverseerAutoScalingTriggerThread-98858489750487044-127.0.0.1:45531_qn-n_0000000000) [n:127.0.0.1:45531_qn    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
   [junit4]   2> 351651 DEBUG (ScheduledTrigger-439-thread-1) [n:127.0.0.1:45531_qn    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:45531_qn]
   [junit4]   2> 352388 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [n:127.0.0.1:45531_qn    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76402bfe
   [junit4]   2> 352549 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [n:127.0.0.1:45531_qn    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76402bfe
   [junit4]   2> 352549 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [n:127.0.0.1:45531_qn    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76402bfe
   [junit4]   2> 352551 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [n:127.0.0.1:45531_qn    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_CB030D4DAD368C76-001/control-001/cores
   [junit4]   2> 352805 DEBUG (ScheduledTrigger-439-thread-1) [n:127.0.0.1:45531_qn    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:45531_qn]
   [junit4]   2> 352916 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 352917 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:36275/solr ready
   [junit4]   2> 352973 INFO  (qtp1303574867-1750) [n:127.0.0.1:45531_qn    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:45531_qn&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 352988 INFO  (OverseerThreadFactory-441-thread-1-processing-n:127.0.0.1:45531_qn) [n:127.0.0.1:45531_qn    ] o.a.s.c.CreateCollectionCmd Create collection control_collection
   [junit4]   2> 353156 INFO  (qtp1303574867-1752) [n:127.0.0.1:45531_qn    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 353156 INFO  (qtp1303574867-1752) [n:127.0.0.1:45531_qn    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 353301 INFO  (zkCallback-406-thread-1-processing-n:127.0.0.1:45531_qn) [n:127.0.0.1:45531_qn    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 353814 DEBUG (ScheduledTrigger-439-thread-2) [n:127.0.0.1:45531_qn    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:45531_qn]
   [junit4]   2> 354528 INFO  (qtp1303574867-1752) [n:127.0.0.1:45531_qn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 354755 INFO  (qtp1303574867-1752) [n:127.0.0.1:45531_qn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
   [junit4]   2> 354818 DEBUG (ScheduledTrigger-439-thread-1) [n:127.0.0.1:45531_qn    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:45531_qn]
   [junit4]   2> 355500 INFO  (qtp1303574867-1752) [n:127.0.0.1:45531_qn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 355875 DEBUG (ScheduledTrigger-439-thread-1) [n:127.0.0.1:45531_qn    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:45531_qn]
   [junit4]   2> 355939 INFO  (qtp1303574867-1752) [n:127.0.0.1:45531_qn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
   [junit4]   2> 355939 INFO  (qtp1303574867-1752) [n:127.0.0.1:45531_qn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76402bfe
   [junit4]   2> 356048 INFO  (qtp1303574867-1752) [n:127.0.0.1:45531_qn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://lucene2-us-west.apache.org:43257/solr_hdfs_home
   [junit4]   2> 356049 INFO  (qtp1303574867-1752) [n:127.0.0.1:45531_qn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
   [junit4]   2> 356049 INFO  (qtp1303574867-1752) [n:127.0.0.1:45531_qn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 356049 INFO  (qtp1303574867-1752) [n:127.0.0.1:45531_qn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_CB030D4DAD368C76-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[hdfs://lucene2-us-west.apache.org:43257/solr_hdfs_home/control_collection/core_node2/data/]
   [junit4]   2> 356050 INFO  (qtp1303574867-1752) [n:127.0.0.1:45531_qn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://lucene2-us-west.apache.org:43257/solr_hdfs_home/control_collection/core_node2/data/snapshot_metadata
   [junit4]   2> 356274 INFO  (qtp1303574867-1752) [n:127.0.0.1:45531_qn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
   [junit4]   2> 356274 INFO  (qtp1303574867-1752) [n:127.0.0.1:45531_qn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [4194304] will allocate [1] slabs and use ~[4194304] bytes
   [junit4]   2> 356274 INFO  (qtp1303574867-1752) [n:127.0.0.1:45531_qn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Creating new global HDFS BlockCache
   [junit4]   2> 356884 DEBUG (ScheduledTrigger-439-thread-1) [n:127.0.0.1:45531_qn    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:45531_qn]
   [junit4]   2> 357893 DEBUG (ScheduledTrigger-439-thread-1) [n:127.0.0.1:45531_qn    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:45531_qn]
   [junit4]   2> 358901 DEBUG (ScheduledTrigger-439-thread-1) [n:127.0.0.1:45531_qn    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:45531_qn]
   [junit4]   2> 359906 DEBUG (ScheduledTrigger-439-thread-1) [n:127.0.0.1:45531_qn    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:45531_qn]
   [junit4]   2> 360630 INFO  (qtp1303574867-1752) [n:127.0.0.1:45531_qn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 360649 INFO  (qtp1303574867-1752) [n:127.0.0.1:45531_qn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://lucene2-us-west.apache.org:43257/solr_hdfs_home/control_collection/core_node2/data
   [junit4]   2> 360913 DEBUG (ScheduledTrigger-439-thread-1) [n:127.0.0.1:45531_qn    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:45531_qn]
   [junit4]   2> 360958 INFO  (qtp1303574867-1752) [n:127.0.0.1:45531_qn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://lucene2-us-west.apache.org:43257/solr_hdfs_home/control_collection/core_node2/data/index
   [junit4]   2> 361031 INFO  (qtp1303574867-1752) [n:127.0.0.1:45531_qn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
   [junit4]   2> 361031 INFO  (qtp1303574867-1752) [n:127.0.0.1:45531_qn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [4194304] will allocate [1] slabs and use ~[4194304] bytes
   [junit4]   2> 361119 INFO  (qtp1303574867-1752) [n:127.0.0.1:45531_qn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 361119 INFO  (qtp1303574867-1752) [n:127.0.0.1:45531_qn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=34, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7080790159303935]
   [junit4]   2> 361922 DEBUG (ScheduledTrigger-439-thread-1) [n:127.0.0.1:45531_qn    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:45531_qn]
   [junit4]   2> 362922 DEBUG (ScheduledTrigger-439-thread-1) [n:127.0.0.1:45531_qn    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:45531_qn]
   [junit4]   2> 363876 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:41645 is added to blk_1073741825_1001{UCState=COMMITTED, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-beeef937-622b-4a78-b64a-35665c55855e:NORMAL:127.0.0.1:45153|RBW], ReplicaUC[[DISK]DS-28837cd2-6ce1-49a1-a162-239e582d448b:NORMAL:127.0.0.1:41645|RBW]]} size 72
   [junit4]   2> 363911 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:45153 is added to blk_1073741825_1001 size 72
   [junit4]   2> 363931 DEBUG (ScheduledTrigger-439-thread-1) [n:127.0.0.1:45531_qn    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:45531_qn]
   [junit4]   2> 364437 WARN  (qtp1303574867-1752) [n:127.0.0.1:45531_qn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 364936 DEBUG (ScheduledTrigger-439-thread-1) [n:127.0.0.1:45531_qn    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:45531_qn]
   [junit4]   2> 365048 INFO  (qtp1303574867-1752) [n:127.0.0.1:45531_qn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
   [junit4]   2> 365048 INFO  (qtp1303574867-1752) [n:127.0.0.1:45531_qn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 365048 INFO  (qtp1303574867-1752) [n:127.0.0.1:45531_qn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
   [junit4]   2> 365242 INFO  (qtp1303574867-1752) [n:127.0.0.1:45531_qn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 365242 INFO  (qtp1303574867-1752) [n:127.0.0.1:45531_qn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 365278 INFO  (qtp1303574867-1752) [n:127.0.0.1:45531_qn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=16, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8399145522025399]
   [junit4]   2> 365944 DEBUG (ScheduledTrigger-439-thread-1) [n:127.0.0.1:45531_qn    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:45531_qn]
   [junit4]   2> 366000 INFO  (qtp1303574867-1752) [n:127.0.0.1:45531_qn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@30b6ae97[control_collection_shard1_replica_n1] main]
   [junit4]   2> 366054 INFO  (qtp1303574867-1752) [n:127.0.0.1:45531_qn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 366070 INFO  (qtp1303574867-1752) [n:127.0.0.1:45531_qn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 366073 INFO  (qtp1303574867-1752) [n:127.0.0.1:45531_qn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 366109 INFO  (searcherExecutor-444-thread-1-processing-n:127.0.0.1:45531_qn x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:45531_qn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@30b6ae97[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 366110 INFO  (qtp1303574867-1752) [n:127.0.0.1:45531_qn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1581735854347386880
   [junit4]   2> 366210 INFO  (qtp1303574867-1752) [n:127.0.0.1:45531_qn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 366210 INFO  (qtp1303574867-1752) [n:127.0.0.1:45531_qn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 366210 INFO  (qtp1303574867-1752) [n:127.0.0.1:45531_qn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:45531/qn/control_collection_shard1_replica_n1/
   [junit4]   2> 366210 INFO  (qtp1303574867-1752) [n:127.0.0.1:45531_qn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 366210 INFO  (qtp1303574867-1752) [n:127.0.0.1:45531_qn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:45531/qn/control_collection_shard1_replica_n1/ has no replicas
   [junit4]   2> 366210 INFO  (qtp1303574867-1752) [n:127.0.0.1:45531_qn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 366213 INFO  (qtp1303574867-1752) [n:127.0.0.1:45531_qn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:45531/qn/control_collection_shard1_replica_n1/ shard1
   [junit4]   2> 366226 INFO  (zkCallback-406-thread-2-processing-n:127.0.0.1:45531_qn) [n:127.0.0.1:45531_qn    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 366282 INFO  (qtp1303574867-1752) [n:127.0.0.1:45531_qn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 366283 INFO  (qtp1303574867-1752) [n:127.0.0.1:45531_qn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=13127
   [junit4]   2> 366412 INFO  (zkCallback-406-thread-2-processing-n:127.0.0.1:45531_qn) [n:127.0.0.1:45531_qn    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 366417 INFO  (qtp1303574867-1750) [n:127.0.0.1:45531_qn    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 366949 DEBUG (ScheduledTrigger-439-thread-1) [n:127.0.0.1:45531_qn    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:45531_qn]
   [junit4]   2> 367222 INFO  (OverseerCollectionConfigSetProcessor-98858489750487044-127.0.0.1:45531_qn-n_0000000000) [n:127.0.0.1:45531_qn    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 367425 INFO  (qtp1303574867-1750) [n:127.0.0.1:45531_qn    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:45531_qn&wt=javabin&version=2} status=0 QTime=14452
   [junit4]   2> 367689 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 367798 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:36275/solr ready
   [junit4]   2> 367799 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 367800 INFO  (qtp1303574867-1751) [n:127.0.0.1:45531_qn    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 367900 INFO  (OverseerThreadFactory-441-thread-2-processing-n:127.0.0.1:45531_qn) [n:127.0.0.1:45531_qn    ] o.a.s.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 367953 WARN  (OverseerThreadFactory-441-thread-2-processing-n:127.0.0.1:45531_qn) [n:127.0.0.1:45531_qn    ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
   [junit4]   2> 367953 DEBUG (ScheduledTrigger-439-thread-1) [n:127.0.0.1:45531_qn    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:45531_qn]
   [junit4]   2> 368392 INFO  (qtp1303574867-1751) [n:127.0.0.1:45531_qn    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 368393 INFO  (qtp1303574867-1751) [n:127.0.0.1:45531_qn    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=592
   [junit4]   2> 368993 DEBUG (ScheduledTrigger-439-thread-1) [n:127.0.0.1:45531_qn    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:45531_qn]
   [junit4]   2> 369414 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_CB030D4DAD368C76-001/shard-1-001 of type NRT
   [junit4]   2> 369414 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 369559 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7fccfc11{/qn,null,AVAILABLE}
   [junit4]   2> 369559 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@365ac6be{HTTP/1.1,[http/1.1]}{127.0.0.1:43473}
   [junit4]   2> 369559 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.e.j.s.Server Started @384922ms
   [junit4]   2> 369559 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/qn, hostPort=43473, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_CB030D4DAD368C76-001/shard-1-001/cores}
   [junit4]   2> 369560 ERROR (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 369560 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 369560 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 369560 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 369560 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-20T00:54:23.080Z
   [junit4]   2> 369706 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 369706 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_CB030D4DAD368C76-001/shard-1-001/solr.xml
   [junit4]   2> 369722 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 369722 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 369743 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@76402bfe, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 369779 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36275/solr
   [junit4]   2> 369928 INFO  (OverseerCollectionConfigSetProcessor-98858489750487044-127.0.0.1:45531_qn-n_0000000000) [n:127.0.0.1:45531_qn    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 370000 DEBUG (ScheduledTrigger-439-thread-1) [n:127.0.0.1:45531_qn    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:45531_qn]
   [junit4]   2> 370091 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [n:127.0.0.1:43473_qn    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 370125 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [n:127.0.0.1:43473_qn    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 370126 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [n:127.0.0.1:43473_qn    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43473_qn
   [junit4]   2> 370144 INFO  (zkCallback-414-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 370234 INFO  (zkCallback-419-thread-1-processing-n:127.0.0.1:43473_qn) [n:127.0.0.1:43473_qn    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 370239 INFO  (zkCallback-406-thread-2-processing-n:127.0.0.1:45531_qn) [n:127.0.0.1:45531_qn    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 371003 DEBUG (ScheduledTrigger-439-thread-1) [n:127.0.0.1:45531_qn    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:43473_qn, 127.0.0.1:45531_qn]
   [junit4]   2> 372268 DEBUG (ScheduledTrigger-439-thread-1) [n:127.0.0.1:45531_qn    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:43473_qn, 127.0.0.1:45531_qn]
   [junit4]   2> 372285 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [n:127.0.0.1:43473_qn    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76402bfe
   [junit4]   2> 372433 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [n:127.0.0.1:43473_qn    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76402bfe
   [junit4]   2> 372433 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [n:127.0.0.1:43473_qn    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76402bfe
   [junit4]   2> 372434 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[CB030D4DAD368C76]) [n:127.0.0.1:43473_qn    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_CB030D4DAD368C76-001/shard-1-001/cores
   [junit4]   2> 372745 INFO  (qtp1501626193-1811) [n:127.0.0.1:43473_qn    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:43473_qn&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 372822 INFO  (OverseerThreadFactory-441-thread-3-processing-n:127.0.0.1:45531_qn) [n:127.0.0.1:45531_qn    ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:43473_qn for creating new replica
   [junit4]   2> 372856 INFO  (qtp1501626193-1809) [n:127.0.0.1:43473_qn    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 372856 INFO  (qtp1501626193-1809) [n:127.0.0.1:43473_qn    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 373186 INFO  (qtp1501626193-1809) [n:127.0.0.1:43473_qn c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 373277 DEBUG (ScheduledTrigger-439-thread-1) [n:127.0.0.1:45531_qn    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:43473_qn, 127.0.0.1:45531_qn]
   [junit4]   2> 373295 INFO  (qtp1501626193-1809) [n:127.0.0.1:43473_qn c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.s.IndexSchema [collection1_shard2_replica_n41] Schema name=test
   [junit4]   2> 374223 INFO  (qtp1501626193-1809) [n:127.0.0.1:43473_qn c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 374293 DEBUG (ScheduledTrigger-439-thread-1) [n:127.0.0.1:45531_qn    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:43473_qn, 127.0.0.1:45531_qn]
   [junit4]   2> 374332 INFO  (qtp1501626193-1809) [n:127.0.0.1:43473_qn c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_n41' using configuration from collection collection1, trusted=true
   [junit4]   2> 374332 INFO  (qtp1501626193-1809) [n:127.0.0.1:43473_qn c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_n41' (registry 'solr.core.collection1.shard2.replica_n41') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76402bfe
   [junit4]   2> 374332 INFO  (qtp1501626193-1809) [n:127.0.0.1:43473_qn c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://lucene2-us-west.apache.org:43257/solr_hdfs_home
   [junit4]   2> 374383 INFO  (qtp1501626193-1809) [n:127.0.0.1:43473_qn c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
   [junit4]   2> 374383 INFO  (qtp1501626193-1809) [n:127.0.0.1:43473_qn c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 374383 INFO  (qtp1501626193-1809) [n:127.0.0.1:43473_qn c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.SolrCore [[collection1_shard2_replica_n41] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_CB030D4DAD368C76-001/shard-1-001/cores/collection1_shard2_replica_n41], dataDir=[hdfs://lucene2-us-west.apache.org:43257/solr_hdfs_home/collection1/core_node42/data/]
   [junit4]   2> 374385 INFO  (qtp1501626193-1809) [n:127.0.0.1:43473_qn c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://lucene2-us-west.apache.org:43257/solr_hdfs_home/collection1/core_node42/data/snapshot_metadata
   [junit4]   2> 374422 INFO  (qtp1501626193-1809) [n:127.0.0.1:43473_qn c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
   [junit4]   2> 374422 INFO  (qtp1501626193-1809) [n:127.0.0.1:43473_qn c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [4194304] will allocate [1] slabs and use ~[4194304] bytes
   [junit4]   2> 374489 INFO  (qtp1501626193-1809) [n:127.0.0.1:43473_qn c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 374562 INFO  (qtp1501626193-1809) [n:127.0.0.1:43473_qn c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://lucene2-us-west.apache.org:43257/solr_hdfs_home/collection1/core_node42/data
   [junit4]   2> 374883 INFO  (qtp1501626193-1809) [n:127.0.0.1:43473_qn c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://lucene2-us-west.apache.org:43257/solr_hdfs_home/collection1/core_node42/data/index
   [junit4]   2> 374937 INFO  (qtp1501626193-1809) [n:127.0.0.1:43473_qn c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
   [junit4]   2> 374937 INFO  (qtp1501626193-1809) [n:127.0.0.1:43473_qn c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [4194304] will allocate [1] slabs and use ~[4194304] bytes
   [junit4]   2> 374989 INFO  (qtp1501626193-1809) [n:127.0.0.1:43473_qn c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 374990 INFO  (qtp1501626193-1809) [n:127.0.0.1:43473_qn c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=34, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7080790159303935]
   [junit4]   2> 375306 DEBUG (ScheduledTrigger-439-thread-1) [n:127.0.0.1:45531_qn    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:43473_qn, 127.0.0.1:45531_qn]
   [junit4]   2> 375577 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:41645 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-ead23820-f5e4-46f7-881c-faf146cf42e4:NORMAL:127.0.0.1:45153|RBW], ReplicaUC[[DISK]DS-1f6d0eca-5556-4c49-8981-9b1d2abe6077:NORMAL:127.0.0.1:41645|RBW]]} size 0
   [junit4]   2> 375597 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:45153 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-1f6d0eca-5556-4c49-8981-9b1d2abe6077:NORMAL:127.0.0.1:41645|RBW], ReplicaUC[[DISK]DS-beeef937-622b-4a78-b64a-35665c55855e:NORMAL:127.0.0.1:45153|FINALIZED]]} size 0
   [junit4]   2> 375842 WARN  (qtp1501626193-1809) [n:127.0.0.1:43473_qn c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 376315 INFO  (qtp1501626193-1809) [n:127.0.0.1:43473_qn c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
   [junit4]   2> 376315 INFO  (qtp1501626193-1809) [n:127.0.0.1:43473_qn c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 376315 DEBUG (ScheduledTrigger-439-thread-1) [n:127.0.0.1:45531_qn    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:43473_qn, 127.0.0.1:45531_qn]
   [junit4]   2> 376315 INFO  (qtp1501626193-1809) [n:127.0.0.1:43473_qn c:collection1 s:shard2  x:collection1

[...truncated too long message...]

lr.collection.collection1.shard1.leader, tag=1150258526
   [junit4]   2> 7037143 INFO  (recoveryExecutor-5524-thread-1-processing-n:127.0.0.1:41480__fpj x:shardSplitWithRule_shard1_0_replica0 s:shard1_0 c:shardSplitWithRule r:core_node10) [n:127.0.0.1:41480__fpj c:shardSplitWithRule s:shard1_0 r:core_node10 x:shardSplitWithRule_shard1_0_replica0] o.a.s.c.RecoveryStrategy Finished recovery process, successful=[false]
   [junit4]   2> 7037145 WARN  (updateExecutor-5523-thread-2-processing-n:127.0.0.1:41480__fpj x:shardSplitWithRule_shard1_0_replica0 s:shard1_0 c:shardSplitWithRule r:core_node10) [n:127.0.0.1:41480__fpj c:shardSplitWithRule s:shard1_0 r:core_node10 x:shardSplitWithRule_shard1_0_replica0] o.a.s.u.DefaultSolrCoreState Skipping recovery because Solr is shutdown
   [junit4]   2> 7037159 INFO  (TEST-ShardSplitTest.testSplitShardWithRule-seed#[CB030D4DAD368C76]) [    ] o.a.s.c.Overseer Overseer (id=98858924721897490-127.0.0.1:41480__fpj-n_0000000004) closing
   [junit4]   2> 7037160 INFO  (OverseerStateUpdate-98858924721897490-127.0.0.1:41480__fpj-n_0000000004) [n:127.0.0.1:41480__fpj    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:41480__fpj
   [junit4]   2> 7037160 WARN  (OverseerAutoScalingTriggerThread-98858924721897490-127.0.0.1:41480__fpj-n_0000000004) [n:127.0.0.1:41480__fpj    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 7037162 INFO  (zkCallback-5526-thread-1-processing-n:127.0.0.1:41480__fpj) [n:127.0.0.1:41480__fpj c:shardSplitWithRule s:shard1_0 r:core_node10 x:shardSplitWithRule_shard1_0_replica0] o.a.s.c.SolrCore [shardSplitWithRule_shard1_0_replica0]  CLOSING SolrCore org.apache.solr.core.SolrCore@15b47362
   [junit4]   2> 7037162 INFO  (zkCallback-5526-thread-1-processing-n:127.0.0.1:41480__fpj) [n:127.0.0.1:41480__fpj c:shardSplitWithRule s:shard1_0 r:core_node10 x:shardSplitWithRule_shard1_0_replica0] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.shardSplitWithRule.shard1_0.replica0, tag=364147554
   [junit4]   2> 7037162 INFO  (zkCallback-5526-thread-1-processing-n:127.0.0.1:41480__fpj) [n:127.0.0.1:41480__fpj c:shardSplitWithRule s:shard1_0 r:core_node10 x:shardSplitWithRule_shard1_0_replica0] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@374e09ec: rootName = null, domain = solr.core.shardSplitWithRule.shard1_0.replica0, service url = null, agent id = null] for registry solr.core.shardSplitWithRule.shard1_0.replica0 / com.codahale.metrics.MetricRegistry@1e864e2c
   [junit4]   2> 7037304 INFO  (zkCallback-5526-thread-1-processing-n:127.0.0.1:41480__fpj) [n:127.0.0.1:41480__fpj c:shardSplitWithRule s:shard1_0 r:core_node10 x:shardSplitWithRule_shard1_0_replica0] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.shardSplitWithRule.shard1_0.leader, tag=364147554
   [junit4]   2> 7037326 WARN  (zkCallback-5526-thread-1-processing-n:127.0.0.1:41480__fpj) [n:127.0.0.1:41480__fpj c:shardSplitWithRule s:shard1_0 r:core_node10 x:shardSplitWithRule_shard1_0_replica0] o.a.s.c.RecoveryStrategy Stopping recovery for core=[shardSplitWithRule_shard1_0_replica0] coreNodeName=[core_node10]
   [junit4]   2> 7037527 INFO  (TEST-ShardSplitTest.testSplitShardWithRule-seed#[CB030D4DAD368C76]) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@13fbc40c{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 7037527 INFO  (TEST-ShardSplitTest.testSplitShardWithRule-seed#[CB030D4DAD368C76]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@7b5aba7e{/_fpj,null,UNAVAILABLE}
   [junit4]   2> 7037588 INFO  (TEST-ShardSplitTest.testSplitShardWithRule-seed#[CB030D4DAD368C76]) [    ] o.a.s.c.ChaosMonkey monkey: stop jetty! 34642
   [junit4]   2> 7037588 ERROR (TEST-ShardSplitTest.testSplitShardWithRule-seed#[CB030D4DAD368C76]) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 7037591 INFO  (TEST-ShardSplitTest.testSplitShardWithRule-seed#[CB030D4DAD368C76]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:40181 40181
   [junit4]   2> 7037747 INFO  (Thread-27634) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:40181 40181
   [junit4]   2> 7037783 WARN  (Thread-27634) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	7	/solr/aliases.json
   [junit4]   2> 	5	/solr/security.json
   [junit4]   2> 	5	/solr/configs/conf1
   [junit4]   2> 	4	/solr/collections/collection1/state.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	68	/solr/collections/shardSplitWithRule/state.json
   [junit4]   2> 	7	/solr/clusterprops.json
   [junit4]   2> 	7	/solr/clusterstate.json
   [junit4]   2> 	5	/solr/autoscaling.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	7	/solr/live_nodes
   [junit4]   2> 	7	/solr/collections
   [junit4]   2> 	5	/solr/overseer/queue
   [junit4]   2> 	5	/solr/autoscaling/events/.auto_add_replicas
   [junit4]   2> 	5	/solr/overseer/collection-queue-work
   [junit4]   2> 	5	/solr/overseer/queue-work
   [junit4]   2> 
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_CB030D4DAD368C76-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {rnd_b=PostingsFormat(name=LuceneFixedGap), shard_s=FST50, n_ti=FST50, a_t=FST50, a_i=PostingsFormat(name=LuceneFixedGap), id=PostingsFormat(name=LuceneFixedGap)}, docValues:{rnd_b=DocValuesFormat(name=Lucene70), shard_s=DocValuesFormat(name=Direct), _version_=DocValuesFormat(name=Lucene70), n_ti=DocValuesFormat(name=Direct), a_t=DocValuesFormat(name=Direct), a_i=DocValuesFormat(name=Lucene70), id=DocValuesFormat(name=Lucene70)}, maxPointsInLeafNode=1685, maxMBSortInHeap=6.549347873454913, sim=RandomSimilarity(queryNorm=true): {}, locale=ar-YE, timezone=Africa/Gaborone
   [junit4]   2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=71636720,total=526385152
   [junit4]   2> NOTE: All tests run in this JVM: [PreAnalyzedFieldTest, TestConfigSetImmutable, EnumFieldTest, HdfsRestartWhileUpdatingTest, TestAuthorizationFramework, TestFieldSortValues, TestPayloadScoreQParserPlugin, SpellingQueryConverterTest, BadIndexSchemaTest, SampleTest, DataDrivenBlockJoinTest, TestRestManager, DocumentAnalysisRequestHandlerTest, TestSubQueryTransformerDistrib, TestCloudRecovery, TestFastWriter, TestPrepRecovery, RollingRestartTest, LeaderFailoverAfterPartitionTest, TestDFISimilarityFactory, TestSolrFieldCacheBean, TestHdfsBackupRestoreCore, TestMacroExpander, NumericFieldsTest, TestSchemalessBufferedUpdates, TestSolrConfigHandlerCloud, TestReplicaProperties, TestAtomicUpdateErrorCases, TestSolrCloudWithKerberosAlt, SolrIndexSplitterTest, ClassificationUpdateProcessorTest, SearchRateTriggerTest, QueryEqualityTest, TriggerIntegrationTest, TestExclusionRuleCollectionAccess, MetricUtilsTest, PrimUtilsTest, TestSurroundQueryParser, PathHierarchyTokenizerFactoryTest, LeaderInitiatedRecoveryOnShardRestartTest, TestNumericTerms32, AnalyticsMergeStrategyTest, LukeRequestHandlerTest, CreateCollectionCleanupTest, AddReplicaTest, ShowFileRequestHandlerTest, CachingDirectoryFactoryTest, CdcrRequestHandlerTest, TestSolrCloudSnapshots, RulesTest, TestSmileRequest, DebugComponentTest, CdcrBootstrapTest, HLLSerializationTest, SolrCoreMetricManagerTest, AddBlockUpdateTest, TestJmxIntegration, TestRTGBase, OutputWriterTest, QueryResultKeyTest, CoreAdminCreateDiscoverTest, TestManagedStopFilterFactory, ShufflingReplicaListTransformerTest, HighlighterConfigTest, SuggestComponentTest, CollectionsAPIDistributedZkTest, TestDocBasedVersionConstraints, SecurityConfHandlerTest, CloudMLTQParserTest, TestStressRecovery, DistributedVersionInfoTest, TestSimpleTextCodec, HdfsNNFailoverTest, TestPerFieldSimilarity, DistributedFacetExistsSmallTest, DistribDocExpirationUpdateProcessorTest, TestXIncludeConfig, ZkStateWriterTest, TestCloudManagedSchema, TestLuceneMatchVersion, SuggesterFSTTest, FacetPivotSmallTest, InfoHandlerTest, TestCollectionAPI, TestSolrQueryParser, TestSha256AuthenticationProvider, TestInPlaceUpdatesDistrib, AssignTest, TestFastLRUCache, TestSolrCloudWithHadoopAuthPlugin, LargeFieldTest, TestCollationField, TestExactSharedStatsCache, NotRequiredUniqueKeyTest, TestCoreAdminApis, TestJettySolrRunner, SolrCLIZkUtilsTest, SearchHandlerTest, TestReloadDeadlock, HdfsTlogReplayBufferedWhileIndexingTest, TestOmitPositions, TestSolrCLIRunExample, TestJavabinTupleStreamParser, SynonymTokenizerTest, CloudExitableDirectoryReaderTest, TestStressInPlaceUpdates, MoreLikeThisHandlerTest, DistributedTermsComponentTest, SortSpecParsingTest, ZkSolrClientTest, TestEmbeddedSolrServerAdminHandler, ChaosMonkeySafeLeaderWithPullReplicasTest, ComputePlanActionTest, ResponseHeaderTest, TestSchemaVersionResource, StressHdfsTest, ShardRoutingTest, ShardSplitTest]
   [junit4] Completed [375/745 (4!)] on J1 in 715.53s, 5 tests, 1 failure <<< FAILURES!

[...truncated 9810 lines...]
BUILD FAILED
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:834: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:778: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid17465.hprof

Total time: 860 minutes 26 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any

[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1403 - Failure

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1403/

13 tests failed.
FAILED:  org.apache.lucene.index.TestIndexSorting.testRandom3

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
	at __randomizedtesting.SeedInfo.seed([4A73DD356EFE1222]:0)


FAILED:  junit.framework.TestSuite.org.apache.lucene.index.TestIndexSorting

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
	at __randomizedtesting.SeedInfo.seed([4A73DD356EFE1222]:0)


FAILED:  org.apache.solr.cloud.ShardSplitTest.testSplitAfterFailedSplit

Error Message:
Shard split did not succeed after a previous failed split attempt left sub-shards in construction state

Stack Trace:
java.lang.AssertionError: Shard split did not succeed after a previous failed split attempt left sub-shards in construction state
	at __randomizedtesting.SeedInfo.seed([F02C81D8640D9F8F:96112775878D205]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.ShardSplitTest.testSplitAfterFailedSplit(ShardSplitTest.java:294)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.autoscaling.AutoAddReplicasIntegrationTest.testSimple

Error Message:
Error from server at https://127.0.0.1:39062/solr: KeeperErrorCode = NoNode for /overseer/collection-queue-work/qnr-0000000004

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:39062/solr: KeeperErrorCode = NoNode for /overseer/collection-queue-work/qnr-0000000004
	at __randomizedtesting.SeedInfo.seed([F02C81D8640D9F8F:C89FA52643FE4B5E]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:626)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1121)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:862)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
	at org.apache.solr.cloud.autoscaling.AutoAddReplicasIntegrationTest.testSimple(AutoAddReplicasIntegrationTest.java:78)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.autoscaling.ComputePlanActionTest.testNodeLost

Error Message:
Trigger was not fired even after 10 seconds

Stack Trace:
java.lang.AssertionError: Trigger was not fired even after 10 seconds
	at __randomizedtesting.SeedInfo.seed([F02C81D8640D9F8F:4F394F26E7E7FA09]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.cloud.autoscaling.ComputePlanActionTest.testNodeLost(ComputePlanActionTest.java:194)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.testEventQueue

Error Message:
action wasn't interrupted

Stack Trace:
java.lang.AssertionError: action wasn't interrupted
	at __randomizedtesting.SeedInfo.seed([F02C81D8640D9F8F:3999C3766D6A597A]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.testEventQueue(TriggerIntegrationTest.java:684)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.security.TestPKIAuthenticationPlugin.test

Error Message:


Stack Trace:
java.lang.NullPointerException
	at __randomizedtesting.SeedInfo.seed([F02C81D8640D9F8F:7878BE02CAF1F277]:0)
	at org.apache.solr.security.TestPKIAuthenticationPlugin.test(TestPKIAuthenticationPlugin.java:100)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.hdfs.HdfsRestartWhileUpdatingTest.test

Error Message:
shard1 is not consistent.  Got 582 from https://127.0.0.1:47806/collection1_shard1_replica_n23 (previous client) and got 337 from https://127.0.0.1:58092/collection1_shard1_replica_n25

Stack Trace:
java.lang.AssertionError: shard1 is not consistent.  Got 582 from https://127.0.0.1:47806/collection1_shard1_replica_n23 (previous client) and got 337 from https://127.0.0.1:58092/collection1_shard1_replica_n25
	at __randomizedtesting.SeedInfo.seed([F02C81D8640D9F8F:7878BE02CAF1F277]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1323)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1302)
	at org.apache.solr.cloud.RestartWhileUpdatingTest.test(RestartWhileUpdatingTest.java:155)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.DistribJoinFromCollectionTest.testNoScore

Error Message:
Error from server at https://127.0.0.1:35489/solr/to_2x2_shard1_replica_n3: Error from server at https://127.0.0.1:57328/solr/to_2x2_shard2_replica_n7: SolrCloud join: Collection 'from_1x4Alias' not found!

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:35489/solr/to_2x2_shard1_replica_n3: Error from server at https://127.0.0.1:57328/solr/to_2x2_shard2_replica_n7: SolrCloud join: Collection 'from_1x4Alias' not found!
	at __randomizedtesting.SeedInfo.seed([F02C81D8640D9F8F:4FA38D6A9A4180C3]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:626)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1121)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:862)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.DistribJoinFromCollectionTest.testJoins(DistribJoinFromCollectionTest.java:174)
	at org.apache.solr.cloud.DistribJoinFromCollectionTest.testNoScore(DistribJoinFromCollectionTest.java:121)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
Didn't see all replicas for shard shard1 in c8n_1x3_lf come up within 90000 ms! ClusterState: {   "control_collection":{     "pullReplicas":"0",     "replicationFactor":"1",     "shards":{"shard1":{         "range":"80000000-7fffffff",         "state":"active",         "replicas":{"core_node2":{             "core":"control_collection_shard1_replica_n1",             "base_url":"http://127.0.0.1:51365",             "node_name":"127.0.0.1:51365_",             "state":"active",             "type":"NRT",             "leader":"true"}}}},     "router":{"name":"compositeId"},     "maxShardsPerNode":"1",     "autoAddReplicas":"false",     "nrtReplicas":"1",     "tlogReplicas":"0"},   "c8n_1x3_lf":{     "pullReplicas":"0",     "replicationFactor":"1",     "shards":{"shard1":{         "range":"80000000-7fffffff",         "state":"active",         "replicas":{           "core_node4":{             "core":"c8n_1x3_lf_shard1_replica_t2",             "base_url":"http://127.0.0.1:58296",             "node_name":"127.0.0.1:58296_",             "state":"active",             "type":"TLOG"},           "core_node5":{             "core":"c8n_1x3_lf_shard1_replica_t3",             "base_url":"http://127.0.0.1:57688",             "node_name":"127.0.0.1:57688_",             "state":"active",             "type":"TLOG",             "leader":"true"},           "core_node6":{             "state":"down",             "base_url":"http://127.0.0.1:44518",             "core":"c8n_1x3_lf_shard1_replica_t1",             "node_name":"127.0.0.1:44518_",             "type":"TLOG"}}}},     "router":{"name":"compositeId"},     "maxShardsPerNode":"1",     "autoAddReplicas":"false",     "nrtReplicas":"0",     "tlogReplicas":"3"},   "collection1":{     "pullReplicas":"0",     "replicationFactor":"1",     "shards":{       "shard1":{         "range":"80000000-ffffffff",         "state":"active",         "replicas":{"core_node44":{             "core":"collection1_shard1_replica_t43",             "base_url":"http://127.0.0.1:58296",             "node_name":"127.0.0.1:58296_",             "state":"active",             "type":"TLOG",             "leader":"true"}}},       "shard2":{         "range":"0-7fffffff",         "state":"active",         "replicas":{           "core_node42":{             "core":"collection1_shard2_replica_t41",             "base_url":"http://127.0.0.1:44518",             "node_name":"127.0.0.1:44518_",             "state":"active",             "type":"TLOG",             "leader":"true"},           "core_node46":{             "core":"collection1_shard2_replica_t45",             "base_url":"http://127.0.0.1:57688",             "node_name":"127.0.0.1:57688_",             "state":"active",             "type":"TLOG"}}}},     "router":{"name":"compositeId"},     "maxShardsPerNode":"1",     "autoAddReplicas":"false",     "nrtReplicas":"1",     "tlogReplicas":"0"}}

Stack Trace:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in c8n_1x3_lf come up within 90000 ms! ClusterState: {
  "control_collection":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "base_url":"http://127.0.0.1:51365",
            "node_name":"127.0.0.1:51365_",
            "state":"active",
            "type":"NRT",
            "leader":"true"}}}},
    "router":{"name":"compositeId"},
    "maxShardsPerNode":"1",
    "autoAddReplicas":"false",
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "c8n_1x3_lf":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node4":{
            "core":"c8n_1x3_lf_shard1_replica_t2",
            "base_url":"http://127.0.0.1:58296",
            "node_name":"127.0.0.1:58296_",
            "state":"active",
            "type":"TLOG"},
          "core_node5":{
            "core":"c8n_1x3_lf_shard1_replica_t3",
            "base_url":"http://127.0.0.1:57688",
            "node_name":"127.0.0.1:57688_",
            "state":"active",
            "type":"TLOG",
            "leader":"true"},
          "core_node6":{
            "state":"down",
            "base_url":"http://127.0.0.1:44518",
            "core":"c8n_1x3_lf_shard1_replica_t1",
            "node_name":"127.0.0.1:44518_",
            "type":"TLOG"}}}},
    "router":{"name":"compositeId"},
    "maxShardsPerNode":"1",
    "autoAddReplicas":"false",
    "nrtReplicas":"0",
    "tlogReplicas":"3"},
  "collection1":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node44":{
            "core":"collection1_shard1_replica_t43",
            "base_url":"http://127.0.0.1:58296",
            "node_name":"127.0.0.1:58296_",
            "state":"active",
            "type":"TLOG",
            "leader":"true"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node42":{
            "core":"collection1_shard2_replica_t41",
            "base_url":"http://127.0.0.1:44518",
            "node_name":"127.0.0.1:44518_",
            "state":"active",
            "type":"TLOG",
            "leader":"true"},
          "core_node46":{
            "core":"collection1_shard2_replica_t45",
            "base_url":"http://127.0.0.1:57688",
            "node_name":"127.0.0.1:57688_",
            "state":"active",
            "type":"TLOG"}}}},
    "router":{"name":"compositeId"},
    "maxShardsPerNode":"1",
    "autoAddReplicas":"false",
    "nrtReplicas":"1",
    "tlogReplicas":"0"}}
	at __randomizedtesting.SeedInfo.seed([F02C81D8640D9F8F:7878BE02CAF1F277]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.ensureAllReplicasAreActive(AbstractFullDistribZkTestBase.java:1996)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:99)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:56)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test

Error Message:
Expected numSlices=5 numReplicas=1 but found DocCollection(solrj_collection3//collections/solrj_collection3/state.json/38)={   "pullReplicas":"0",   "replicationFactor":"1",   "shards":{     "shard1":{       "range":"80000000-b332ffff",       "state":"active",       "replicas":{"core_node4":{           "dataDir":"hdfs://localhost:53778/solr_hdfs_home/solrj_collection3/core_node4/data/",           "base_url":"http://127.0.0.1:44325/ddj",           "node_name":"127.0.0.1:44325_ddj",           "type":"NRT",           "ulogDir":"hdfs://localhost:53778/solr_hdfs_home/solrj_collection3/core_node4/data/tlog",           "core":"solrj_collection3_shard1_replica_n1",           "state":"active",           "leader":"true"}}},     "shard2":{       "range":"b3330000-e665ffff",       "state":"active",       "replicas":{"core_node6":{           "dataDir":"hdfs://localhost:53778/solr_hdfs_home/solrj_collection3/core_node6/data/",           "base_url":"http://127.0.0.1:36086/ddj",           "node_name":"127.0.0.1:36086_ddj",           "type":"NRT",           "ulogDir":"hdfs://localhost:53778/solr_hdfs_home/solrj_collection3/core_node6/data/tlog",           "core":"solrj_collection3_shard2_replica_n2",           "state":"active",           "leader":"true"}}},     "shard3":{       "range":"e6660000-1998ffff",       "state":"active",       "replicas":{}},     "shard4":{       "range":"19990000-4ccbffff",       "state":"active",       "replicas":{"core_node9":{           "dataDir":"hdfs://localhost:53778/solr_hdfs_home/solrj_collection3/core_node9/data/",           "base_url":"http://127.0.0.1:44325/ddj",           "node_name":"127.0.0.1:44325_ddj",           "type":"NRT",           "ulogDir":"hdfs://localhost:53778/solr_hdfs_home/solrj_collection3/core_node9/data/tlog",           "core":"solrj_collection3_shard4_replica_n5",           "state":"active",           "leader":"true"}}},     "shard5":{       "range":"4ccc0000-7fffffff",       "state":"active",       "replicas":{"core_node10":{           "dataDir":"hdfs://localhost:53778/solr_hdfs_home/solrj_collection3/core_node10/data/",           "base_url":"http://127.0.0.1:44325/ddj",           "node_name":"127.0.0.1:44325_ddj",           "type":"NRT",           "ulogDir":"hdfs://localhost:53778/solr_hdfs_home/solrj_collection3/core_node10/data/tlog",           "core":"solrj_collection3_shard5_replica_n7",           "state":"active",           "leader":"true"}}}},   "router":{     "field":"myOwnField",     "name":"compositeId"},   "maxShardsPerNode":"1",   "autoAddReplicas":"true",   "nrtReplicas":"1",   "tlogReplicas":"0"} with /live_nodes: [127.0.0.1:52576_ddj, 127.0.0.1:37135_ddj, 127.0.0.1:44325_ddj, 127.0.0.1:36086_ddj, 127.0.0.1:37789_ddj]

Stack Trace:
java.lang.AssertionError: Expected numSlices=5 numReplicas=1 but found DocCollection(solrj_collection3//collections/solrj_collection3/state.json/38)={
  "pullReplicas":"0",
  "replicationFactor":"1",
  "shards":{
    "shard1":{
      "range":"80000000-b332ffff",
      "state":"active",
      "replicas":{"core_node4":{
          "dataDir":"hdfs://localhost:53778/solr_hdfs_home/solrj_collection3/core_node4/data/",
          "base_url":"http://127.0.0.1:44325/ddj",
          "node_name":"127.0.0.1:44325_ddj",
          "type":"NRT",
          "ulogDir":"hdfs://localhost:53778/solr_hdfs_home/solrj_collection3/core_node4/data/tlog",
          "core":"solrj_collection3_shard1_replica_n1",
          "state":"active",
          "leader":"true"}}},
    "shard2":{
      "range":"b3330000-e665ffff",
      "state":"active",
      "replicas":{"core_node6":{
          "dataDir":"hdfs://localhost:53778/solr_hdfs_home/solrj_collection3/core_node6/data/",
          "base_url":"http://127.0.0.1:36086/ddj",
          "node_name":"127.0.0.1:36086_ddj",
          "type":"NRT",
          "ulogDir":"hdfs://localhost:53778/solr_hdfs_home/solrj_collection3/core_node6/data/tlog",
          "core":"solrj_collection3_shard2_replica_n2",
          "state":"active",
          "leader":"true"}}},
    "shard3":{
      "range":"e6660000-1998ffff",
      "state":"active",
      "replicas":{}},
    "shard4":{
      "range":"19990000-4ccbffff",
      "state":"active",
      "replicas":{"core_node9":{
          "dataDir":"hdfs://localhost:53778/solr_hdfs_home/solrj_collection3/core_node9/data/",
          "base_url":"http://127.0.0.1:44325/ddj",
          "node_name":"127.0.0.1:44325_ddj",
          "type":"NRT",
          "ulogDir":"hdfs://localhost:53778/solr_hdfs_home/solrj_collection3/core_node9/data/tlog",
          "core":"solrj_collection3_shard4_replica_n5",
          "state":"active",
          "leader":"true"}}},
    "shard5":{
      "range":"4ccc0000-7fffffff",
      "state":"active",
      "replicas":{"core_node10":{
          "dataDir":"hdfs://localhost:53778/solr_hdfs_home/solrj_collection3/core_node10/data/",
          "base_url":"http://127.0.0.1:44325/ddj",
          "node_name":"127.0.0.1:44325_ddj",
          "type":"NRT",
          "ulogDir":"hdfs://localhost:53778/solr_hdfs_home/solrj_collection3/core_node10/data/tlog",
          "core":"solrj_collection3_shard5_replica_n7",
          "state":"active",
          "leader":"true"}}}},
  "router":{
    "field":"myOwnField",
    "name":"compositeId"},
  "maxShardsPerNode":"1",
  "autoAddReplicas":"true",
  "nrtReplicas":"1",
  "tlogReplicas":"0"} with /live_nodes: [127.0.0.1:52576_ddj, 127.0.0.1:37135_ddj, 127.0.0.1:44325_ddj, 127.0.0.1:36086_ddj, 127.0.0.1:37789_ddj]
	at __randomizedtesting.SeedInfo.seed([F02C81D8640D9F8F:7878BE02CAF1F277]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.assertSliceAndReplicaCount(SharedFSAutoReplicaFailoverTest.java:413)
	at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:271)
	at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:143)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.TestStressLiveNodes.testStress

Error Message:
iter1851 6 != 1 expected:<[127.0.0.1:54041_solr, thrasher-T1851_0-0, thrasher-T1851_1-0, thrasher-T1851_2-0, thrasher-T1851_3-0, thrasher-T1851_4-0]> but was:<[127.0.0.1:54041_solr]>

Stack Trace:
java.lang.AssertionError: iter1851 6 != 1 expected:<[127.0.0.1:54041_solr, thrasher-T1851_0-0, thrasher-T1851_1-0, thrasher-T1851_2-0, thrasher-T1851_3-0, thrasher-T1851_4-0]> but was:<[127.0.0.1:54041_solr]>
	at __randomizedtesting.SeedInfo.seed([F02C81D8640D9F8F:E537FC19E7FA67F4]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.apache.solr.cloud.TestStressLiveNodes.testStress(TestStressLiveNodes.java:203)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates.test_dv_stored

Error Message:
java.lang.RuntimeException: Could not load collection from ZK: test_col

Stack Trace:
java.util.concurrent.ExecutionException: java.lang.RuntimeException: Could not load collection from ZK: test_col
	at __randomizedtesting.SeedInfo.seed([F02C81D8640D9F8F:6210E05CD84AA257]:0)
	at java.util.concurrent.FutureTask.report(FutureTask.java:122)
	at java.util.concurrent.FutureTask.get(FutureTask.java:192)
	at org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates.checkField(TestStressCloudBlindAtomicUpdates.java:283)
	at org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates.test_dv_stored(TestStressCloudBlindAtomicUpdates.java:205)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.RuntimeException: Could not load collection from ZK: test_col
	at org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates$Worker.run(TestStressCloudBlindAtomicUpdates.java:411)
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	... 1 more
Caused by: org.apache.solr.common.SolrException: Could not load collection from ZK: test_col
	at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1170)
	at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:690)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.getDocCollection(CloudSolrClient.java:1227)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:827)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
	at org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates$Worker.doRandomAtomicUpdate(TestStressCloudBlindAtomicUpdates.java:370)
	at org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates$Worker.run(TestStressCloudBlindAtomicUpdates.java:399)
	... 6 more
Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/test_col/state.json
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
	at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212)
	at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:357)
	at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:354)
	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
	at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)
	at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1182)
	at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1168)
	... 14 more




Build Log:
[...truncated 867 lines...]
   [junit4] Suite: org.apache.lucene.index.TestIndexSorting
   [junit4]   2> Οκτ 16, 2017 10:52:47 ΜΜ com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
   [junit4]   2> WARNING: Suite execution timed out: org.apache.lucene.index.TestIndexSorting
   [junit4]   2>    1) Thread[id=1, name=main, state=WAITING, group=main]
   [junit4]   2>         at java.lang.Object.wait(Native Method)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1252)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1326)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:636)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:493)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:251)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
   [junit4]   2>    2) Thread[id=155, name=SUITE-TestIndexSorting-seed#[4A73DD356EFE1222], state=RUNNABLE, group=TGRP-TestIndexSorting]
   [junit4]   2>         at java.lang.Thread.getStackTrace(Thread.java:1559)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:696)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:693)
   [junit4]   2>         at java.security.AccessController.doPrivileged(Native Method)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getStackTrace(ThreadLeakControl.java:693)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getThreadsWithTraces(ThreadLeakControl.java:709)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.formatThreadStacksFull(ThreadLeakControl.java:689)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.access$1000(ThreadLeakControl.java:65)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$2.evaluate(ThreadLeakControl.java:415)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:705)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.access$200(RandomizedRunner.java:139)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$2.run(RandomizedRunner.java:626)
   [junit4]   2>    3) Thread[id=12288, name=Lucene Merge Thread #11372, state=RUNNABLE, group=TGRP-TestIndexSorting]
   [junit4]   2>         at java.lang.Thread.yield(Native Method)
   [junit4]   2>         at org.apache.lucene.store.MockIndexOutputWrapper.writeBytes(MockIndexOutputWrapper.java:138)
   [junit4]   2>         at org.apache.lucene.store.MockIndexOutputWrapper.writeByte(MockIndexOutputWrapper.java:126)
   [junit4]   2>         at org.apache.lucene.store.RateLimitedIndexOutput.writeByte(RateLimitedIndexOutput.java:66)
   [junit4]   2>         at org.apache.lucene.store.DataOutput.writeSignedVLong(DataOutput.java:233)
   [junit4]   2>         at org.apache.lucene.store.DataOutput.writeZLong(DataOutput.java:246)
   [junit4]   2>         at org.apache.lucene.util.packed.MonotonicBlockPackedWriter.flush(MonotonicBlockPackedWriter.java:92)
   [junit4]   2>         at org.apache.lucene.util.packed.AbstractBlockPackedWriter.add(AbstractBlockPackedWriter.java:80)
   [junit4]   2>         at org.apache.lucene.util.packed.MonotonicBlockPackedWriter.add(MonotonicBlockPackedWriter.java:69)
   [junit4]   2>         at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.addSortedNumericField(MemoryDocValuesConsumer.java:442)
   [junit4]   2>         at org.apache.lucene.codecs.DocValuesConsumer.mergeSortedNumericField(DocValuesConsumer.java:371)
   [junit4]   2>         at org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:143)
   [junit4]   2>         at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:151)
   [junit4]   2>         at org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:181)
   [junit4]   2>         at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:125)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4390)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4030)
   [junit4]   2>         at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:624)
   [junit4]   2>         at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:661)
   [junit4]   2>    4) Thread[id=9, name=JUnit4-serializer-daemon, state=TIMED_WAITING, group=main]
   [junit4]   2>         at java.lang.Thread.sleep(Native Method)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:50)
   [junit4]   2>    5) Thread[id=156, name=TEST-TestIndexSorting.testRandom3-seed#[4A73DD356EFE1222], state=RUNNABLE, group=TGRP-TestIndexSorting]
   [junit4]   2>         at java.lang.Thread.start0(Native Method)
   [junit4]   2>         at java.lang.Thread.start(Thread.java:717)
   [junit4]   2>         at org.apache.lucene.index.ConcurrentMergeScheduler.merge(ConcurrentMergeScheduler.java:553)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2233)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5045)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1718)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1460)
   [junit4]   2>         at org.apache.lucene.index.TestIndexSorting.testRandom3(TestIndexSorting.java:2301)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestIndexSorting -Dtests.method=testRandom3 -Dtests.seed=4A73DD356EFE1222 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=el-GR -Dtests.timezone=US/Indiana-Starke -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   6944s J1 | TestIndexSorting.testRandom3 <<<
   [junit4]    > Throwable #1: java.lang.Exception: Test abandoned because suite timeout was reached.
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([4A73DD356EFE1222]:0)
   [junit4]   2> Οκτ 16, 2017 10:52:47 ΜΜ com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 2 leaked thread(s).
   [junit4]   2> Οκτ 16, 2017 10:53:08 ΜΜ com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 3 threads leaked from SUITE scope at org.apache.lucene.index.TestIndexSorting: 
   [junit4]   2>    1) Thread[id=12288, name=Lucene Merge Thread #11372, state=BLOCKED, group=TGRP-TestIndexSorting]
   [junit4]   2>         at org.apache.lucene.store.MockDirectoryWrapper.maybeThrowDeterministicException(MockDirectoryWrapper.java:1019)
   [junit4]   2>         at org.apache.lucene.store.MockIndexOutputWrapper.writeBytes(MockIndexOutputWrapper.java:144)
   [junit4]   2>         at org.apache.lucene.store.MockIndexOutputWrapper.writeByte(MockIndexOutputWrapper.java:126)
   [junit4]   2>         at org.apache.lucene.store.RateLimitedIndexOutput.writeByte(RateLimitedIndexOutput.java:66)
   [junit4]   2>         at org.apache.lucene.codecs.lucene70.Lucene70DocValuesConsumer.writeBlock(Lucene70DocValuesConsumer.java:326)
   [junit4]   2>         at org.apache.lucene.codecs.lucene70.Lucene70DocValuesConsumer.writeValuesMultipleBlocks(Lucene70DocValuesConsumer.java:294)
   [junit4]   2>         at org.apache.lucene.codecs.lucene70.Lucene70DocValuesConsumer.writeValues(Lucene70DocValuesConsumer.java:261)
   [junit4]   2>         at org.apache.lucene.codecs.lucene70.Lucene70DocValuesConsumer.addSortedNumericField(Lucene70DocValuesConsumer.java:530)
   [junit4]   2>         at org.apache.lucene.codecs.DocValuesConsumer.mergeSortedNumericField(DocValuesConsumer.java:371)
   [junit4]   2>         at org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:143)
   [junit4]   2>         at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:151)
   [junit4]   2>         at org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:181)
   [junit4]   2>         at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:125)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4390)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4030)
   [junit4]   2>         at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:624)
   [junit4]   2>         at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:661)
   [junit4]   2>    2) Thread[id=12318, name=Lucene Merge Thread #11402, state=RUNNABLE, group=TGRP-TestIndexSorting]
   [junit4]   2>         at java.lang.Thread.yield(Native Method)
   [junit4]   2>         at org.apache.lucene.store.MockDirectoryWrapper.maybeYield(MockDirectoryWrapper.java:626)
   [junit4]   2>         at org.apache.lucene.store.MockDirectoryWrapper.openInput(MockDirectoryWrapper.java:745)
   [junit4]   2>         at org.apache.lucene.store.Directory.openChecksumInput(Directory.java:119)
   [junit4]   2>         at org.apache.lucene.store.MockDirectoryWrapper.openChecksumInput(MockDirectoryWrapper.java:1072)
   [junit4]   2>         at org.apache.lucene.codecs.lucene50.Lucene50CompoundReader.readEntries(Lucene50CompoundReader.java:105)
   [junit4]   2>         at org.apache.lucene.codecs.lucene50.Lucene50CompoundReader.<init>(Lucene50CompoundReader.java:69)
   [junit4]   2>         at org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.getCompoundReader(Lucene50CompoundFormat.java:71)
   [junit4]   2>         at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:99)
   [junit4]   2>         at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:78)
   [junit4]   2>         at org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:208)
   [junit4]   2>         at org.apache.lucene.index.ReadersAndUpdates.getReaderForMerge(ReadersAndUpdates.java:836)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4360)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4030)
   [junit4]   2>         at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:624)
   [junit4]   2>         at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:661)
   [junit4]   2>    3) Thread[id=156, name=TEST-TestIndexSorting.testRandom3-seed#[4A73DD356EFE1222], state=BLOCKED, group=TGRP-TestIndexSorting]
   [junit4]   2>         at org.apache.lucene.store.MockDirectoryWrapper.maybeThrowDeterministicException(MockDirectoryWrapper.java:1019)
   [junit4]   2>         at org.apache.lucene.store.MockIndexOutputWrapper.writeBytes(MockIndexOutputWrapper.java:144)
   [junit4]   2>         at org.apache.lucene.store.MockIndexOutputWrapper.writeByte(MockIndexOutputWrapper.java:126)
   [junit4]   2>         at org.apache.lucene.store.DataOutput.writeInt(DataOutput.java:70)
   [junit4]   2>         at org.apache.lucene.store.DataOutput.writeLong(DataOutput.java:212)
   [junit4]   2>         at org.apache.lucene.util.packed.DirectMonotonicWriter.flush(DirectMonotonicWriter.java:84)
   [junit4]   2>         at org.apache.lucene.util.packed.DirectMonotonicWriter.finish(DirectMonotonicWriter.java:127)
   [junit4]   2>         at org.apache.lucene.codecs.lucene70.Lucene70DocValuesConsumer.addSortedNumericField(Lucene70DocValuesConsumer.java:549)
   [junit4]   2>         at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addSortedNumericField(PerFieldDocValuesFormat.java:124)
   [junit4]   2>         at org.apache.lucene.index.SortedNumericDocValuesWriter.flush(SortedNumericDocValuesWriter.java:155)
   [junit4]   2>         at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:247)
   [junit4]   2>         at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:131)
   [junit4]   2>         at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:451)
   [junit4]   2>         at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:542)
   [junit4]   2>         at org.apache.lucene.index.DocumentsWriter.postUpdate(DocumentsWriter.java:399)
   [junit4]   2>         at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:502)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1715)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1460)
   [junit4]   2>         at org.apache.lucene.index.TestIndexSorting.testRandom3(TestIndexSorting.java:2301)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Οκτ 16, 2017 10:53:08 ΜΜ com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=12288, name=Lucene Merge Thread #11372, state=RUNNABLE, group=TGRP-TestIndexSorting]
   [junit4]   2>    2) Thread[id=12318, name=Lucene Merge Thread #11402, state=RUNNABLE, group=TGRP-TestIndexSorting]
   [junit4]   2>    3) Thread[id=156, name=TEST-TestIndexSorting.testRandom3-seed#[4A73DD356EFE1222], state=RUNNABLE, group=TGRP-TestIndexSorting]
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J1/temp/lucene.index.TestIndexSorting_4A73DD356EFE1222-001
   [junit4]   2> Oct 17, 2017 3:53:08 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: All leaked threads terminated.
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {sparse_text=FSTOrd50, docs=PostingsFormat(name=LuceneFixedGap), norms=PostingsFormat(name=LuceneFixedGap), positions=FSTOrd50, id=PostingsFormat(name=LuceneFixedGap), term_vectors=FST50}, docValues:{multi_valued_long=DocValuesFormat(name=Asserting), foo=DocValuesFormat(name=Asserting), numeric=DocValuesFormat(name=Memory), multi_valued_numeric=DocValuesFormat(name=Lucene70), float=DocValuesFormat(name=Lucene70), long=DocValuesFormat(name=Lucene70), points=DocValuesFormat(name=Lucene70), sparse_int=DocValuesFormat(name=Lucene70), bar=DocValuesFormat(name=Lucene70), multi_valued_double=DocValuesFormat(name=Lucene70), docs=DocValuesFormat(name=Lucene70), id=DocValuesFormat(name=Lucene70), multi_valued_int=DocValuesFormat(name=Memory), multi_valued_bytes=DocValuesFormat(name=Memory), sparse_text=DocValuesFormat(name=Asserting), double=DocValuesFormat(name=Memory), sparse_binary=DocValuesFormat(name=Asserting), positions=DocValuesFormat(name=Asserting), int=DocValuesFormat(name=Lucene70), sorted=DocValuesFormat(name=Memory), sparse=DocValuesFormat(name=Asserting), multi_valued_string=DocValuesFormat(name=Lucene70), norms=DocValuesFormat(name=Lucene70), bytes=Doc

[...truncated too long message...]

ut/solr/core/test-lib/apacheds-interceptors-journal-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-normalization-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-operational-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-referral-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-schema-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-subtree-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-trigger-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-jdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-jdbm1-2.0.0-M2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-kerberos-codec-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-ldif-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-mavibot-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-protocol-kerberos-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-protocol-ldap-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-protocol-shared-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-xdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/api-all-1.0.0-M20.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/bcprov-jdk15-1.45.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/byte-buddy-1.6.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/commons-collections-3.2.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/commons-math3-3.6.1.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/ehcache-core-2.4.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/hadoop-common-2.7.4-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/hadoop-hdfs-2.7.4-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/hadoop-minikdc-2.7.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/htrace-core-3.2.0-incubating.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jersey-core-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jersey-server-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jetty-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jetty-sslengine-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jetty-util-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/mina-core-2.0.0-M5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/mockito-core-2.6.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/netty-all-4.0.36.Final.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/objenesis-2.5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/analysis/icu/lucene-analyzers-icu-8.0.0-SNAPSHOT.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/contrib/solr-analysis-extras/classes/java:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/queryparser/classes/test:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/backward-codecs/classes/test:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/contrib/analysis-extras/lib/icu4j-59.1.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-launcher.jar:/x1/jenkins/.ant/lib/ivy-2.3.0.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-log4j.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit4.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jai.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-javamail.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bsf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-net.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-antlr.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jsch.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-oro.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-logging.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-netrexx.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-testutil.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jdepend.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bcel.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-xalan2.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-resolver.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jmf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-regexp.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-swing.jar:/usr/local/asfpackages/java/jdk1.8.0_144/lib/tools.jar:/x1/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.5.3.jar com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J1-20171017_130939_4662332864117658623401.events @/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J1-20171017_130939_4666627597082978853301.suites -stdin
   [junit4] ERROR: JVM J1 ended with an exception: Forked process returned with error code: 1. Very likely a JVM crash.  See process stdout at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J1-20171017_130939_4666797797990569194998.sysout See process stderr at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J1-20171017_130939_4666445489359992254434.syserr
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1519)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:126)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:982)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:979)
   [junit4] 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4] 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4] 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4] 	at java.lang.Thread.run(Thread.java:748)

BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:834: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:770: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:59: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build.xml:262: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/common-build.xml:556: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/common-build.xml:1494: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/common-build.xml:1021: At least one slave process threw an exception, first: Forked process returned with error code: 1. Very likely a JVM crash.  See process stdout at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J0-20171017_130939_4614007999479722110771.sysout See process stderr at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J0-20171017_130939_4612143799229928065402.syserr

Total time: 1094 minutes 38 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[Fast Archiver] No artifacts from Lucene-Solr-NightlyTests-master #1402 to compare, so performing full copy of artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any

[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1402 - Still Unstable

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1402/

9 tests failed.
FAILED:  org.apache.solr.cloud.ForceLeaderTest.testReplicasInLIRNoLeader

Error Message:
expected:<2> but was:<3>

Stack Trace:
java.lang.AssertionError: expected:<2> but was:<3>
	at __randomizedtesting.SeedInfo.seed([174C2DA04E69B19C:F1DB196077EB48FD]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.cloud.ForceLeaderTest.testReplicasInLIRNoLeader(ForceLeaderTest.java:110)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ShardRoutingTest

Error Message:
1 thread leaked from SUITE scope at org.apache.solr.cloud.ShardRoutingTest:     1) Thread[id=7721, name=zkCallback-782-thread-1, state=TIMED_WAITING, group=TGRP-ShardRoutingTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.ShardRoutingTest: 
   1) Thread[id=7721, name=zkCallback-782-thread-1, state=TIMED_WAITING, group=TGRP-ShardRoutingTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([174C2DA04E69B19C]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ShardRoutingTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=7721, name=zkCallback-782-thread-1, state=TIMED_WAITING, group=TGRP-ShardRoutingTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=7721, name=zkCallback-782-thread-1, state=TIMED_WAITING, group=TGRP-ShardRoutingTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([174C2DA04E69B19C]:0)


FAILED:  org.apache.solr.update.SoftAutoCommitTest.testSoftCommitWithinAndHardCommitMaxTimeRapidAdds

Error Message:
softCommit: did not find a single commit

Stack Trace:
java.lang.AssertionError: softCommit: did not find a single commit
	at __randomizedtesting.SeedInfo.seed([174C2DA04E69B19C:71012532B74E0634]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNotNull(Assert.java:526)
	at org.apache.solr.update.SoftAutoCommitTest.assertRapidMultiCommitQueues(SoftAutoCommitTest.java:494)
	at org.apache.solr.update.SoftAutoCommitTest.doTestSoftAndHardCommitMaxTimeRapidAdds(SoftAutoCommitTest.java:459)
	at org.apache.solr.update.SoftAutoCommitTest.testSoftCommitWithinAndHardCommitMaxTimeRapidAdds(SoftAutoCommitTest.java:408)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.update.SoftAutoCommitTest.testHardCommitWithinAndSoftCommitMaxTimeRapidAdds

Error Message:
softCommit: commit#1 has TS too low relative to previous TS & commitWait: commitNanos=8253468924638738, prevTimestampNanos=8253468924683573, commitWaitMillis=500

Stack Trace:
java.lang.AssertionError: softCommit: commit#1 has TS too low relative to previous TS & commitWait: commitNanos=8253468924638738, prevTimestampNanos=8253468924683573, commitWaitMillis=500
	at __randomizedtesting.SeedInfo.seed([174C2DA04E69B19C:82E28BF7AC7BC3E1]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.update.SoftAutoCommitTest.assertRapidMultiCommitQueues(SoftAutoCommitTest.java:498)
	at org.apache.solr.update.SoftAutoCommitTest.doTestSoftAndHardCommitMaxTimeRapidAdds(SoftAutoCommitTest.java:459)
	at org.apache.solr.update.SoftAutoCommitTest.testHardCommitWithinAndSoftCommitMaxTimeRapidAdds(SoftAutoCommitTest.java:411)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.BasicDistributedZkTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:43336

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:43336
	at __randomizedtesting.SeedInfo.seed([174C2DA04E69B19C:9F18127AE095DC64]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:637)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1121)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:862)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
	at org.apache.solr.cloud.BasicDistributedZkTest.testANewCollectionInOneInstanceWithManualShardAssignement(BasicDistributedZkTest.java:801)
	at org.apache.solr.cloud.BasicDistributedZkTest.test(BasicDistributedZkTest.java:374)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:171)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:525)
	... 52 more


FAILED:  org.apache.solr.cloud.hdfs.HdfsSyncSliceTest.test

Error Message:
Timeout occured while waiting response from server at: https://127.0.0.1:41341/ah_/y/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:41341/ah_/y/collection1
	at __randomizedtesting.SeedInfo.seed([174C2DA04E69B19C:9F18127AE095DC64]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:637)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1121)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:862)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:484)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:463)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.commit(AbstractFullDistribZkTestBase.java:1583)
	at org.apache.solr.cloud.SyncSliceTest.test(SyncSliceTest.java:182)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:171)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
	at sun.security.ssl.InputRecord.read(InputRecord.java:503)
	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:983)
	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:940)
	at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:525)
	... 53 more


FAILED:  org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.testNodeMarkersRegistration

Error Message:
Path /autoscaling/nodeAdded/127.0.0.1:46772_solr wasn't created

Stack Trace:
java.lang.AssertionError: Path /autoscaling/nodeAdded/127.0.0.1:46772_solr wasn't created
	at __randomizedtesting.SeedInfo.seed([174C2DA04E69B19C:FF6A5AC405C7C73]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.testNodeMarkersRegistration(TriggerIntegrationTest.java:909)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.hdfs.HdfsTlogReplayBufferedWhileIndexingTest.test

Error Message:
There are still nodes recoverying - waited for 600 seconds

Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 600 seconds
	at __randomizedtesting.SeedInfo.seed([174C2DA04E69B19C:9F18127AE095DC64]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:185)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:914)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1471)
	at org.apache.solr.cloud.TlogReplayBufferedWhileIndexingTest.test(TlogReplayBufferedWhileIndexingTest.java:99)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 13055 lines...]
   [junit4] Suite: org.apache.solr.cloud.ForceLeaderTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ForceLeaderTest_174C2DA04E69B19C-001/init-core-data-001
   [junit4]   2> 401843 WARN  (SUITE-ForceLeaderTest-seed#[174C2DA04E69B19C]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=4 numCloses=4
   [junit4]   2> 401970 INFO  (SUITE-ForceLeaderTest-seed#[174C2DA04E69B19C]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 401972 INFO  (SUITE-ForceLeaderTest-seed#[174C2DA04E69B19C]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
   [junit4]   2> 401972 INFO  (SUITE-ForceLeaderTest-seed#[174C2DA04E69B19C]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 401972 INFO  (SUITE-ForceLeaderTest-seed#[174C2DA04E69B19C]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /gqj/
   [junit4]   2> 402055 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 402055 INFO  (Thread-149) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 402055 INFO  (Thread-149) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 402352 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.c.ZkTestServer start zk server on port:35064
   [junit4]   2> 402566 ERROR (Thread-149) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 402783 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 402784 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 402786 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 402786 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 402787 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 402788 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 402789 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 402789 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 402790 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 402791 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 402791 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 402806 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Will use TLOG replicas unless explicitly asked otherwise
   [junit4]   2> 405785 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 405817 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3d4af5ec{/gqj,null,AVAILABLE}
   [junit4]   2> 405827 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@2ca8560c{HTTP/1.1,[http/1.1]}{127.0.0.1:41796}
   [junit4]   2> 405827 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.e.j.s.Server Started @426304ms
   [junit4]   2> 405827 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ForceLeaderTest_174C2DA04E69B19C-001/tempDir-001/control/data, replicaType=NRT, hostContext=/gqj, hostPort=35456, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ForceLeaderTest_174C2DA04E69B19C-001/control-001/cores}
   [junit4]   2> 405827 ERROR (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 405827 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 405827 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 405828 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 405828 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-14T16:53:42.682Z
   [junit4]   2> 406174 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 406174 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ForceLeaderTest_174C2DA04E69B19C-001/control-001/solr.xml
   [junit4]   2> 406283 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 406283 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 406287 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@48908242, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 406351 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:35064/solr
   [junit4]   2> 407135 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [n:127.0.0.1:35456_gqj    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 407154 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [n:127.0.0.1:35456_gqj    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:35456_gqj
   [junit4]   2> 407155 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [n:127.0.0.1:35456_gqj    ] o.a.s.c.Overseer Overseer (id=98828289239678980-127.0.0.1:35456_gqj-n_0000000000) starting
   [junit4]   2> 408105 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [n:127.0.0.1:35456_gqj    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35456_gqj
   [junit4]   2> 408126 INFO  (zkCallback-126-thread-1-processing-n:127.0.0.1:35456_gqj) [n:127.0.0.1:35456_gqj    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 410915 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [n:127.0.0.1:35456_gqj    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@48908242
   [junit4]   2> 410935 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [n:127.0.0.1:35456_gqj    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@48908242
   [junit4]   2> 410935 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [n:127.0.0.1:35456_gqj    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@48908242
   [junit4]   2> 410936 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [n:127.0.0.1:35456_gqj    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ForceLeaderTest_174C2DA04E69B19C-001/control-001/cores
   [junit4]   2> 411396 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 411396 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:35064/solr ready
   [junit4]   2> 411398 INFO  (SocketProxy-Acceptor-35456) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=52808,localport=35456], receiveBufferSize:531000
   [junit4]   2> 411416 INFO  (SocketProxy-Acceptor-35456) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=41796,localport=57110], receiveBufferSize=530904
   [junit4]   2> 411448 INFO  (qtp83303268-705) [n:127.0.0.1:35456_gqj    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:35456_gqj&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 411456 INFO  (OverseerThreadFactory-284-thread-1-processing-n:127.0.0.1:35456_gqj) [n:127.0.0.1:35456_gqj    ] o.a.s.c.CreateCollectionCmd Create collection control_collection
   [junit4]   2> 411889 INFO  (SocketProxy-Acceptor-35456) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=52813,localport=35456], receiveBufferSize:531000
   [junit4]   2> 412063 INFO  (qtp83303268-703) [n:127.0.0.1:35456_gqj    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 412064 INFO  (qtp83303268-703) [n:127.0.0.1:35456_gqj    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 412067 INFO  (SocketProxy-Acceptor-35456) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=41796,localport=57115], receiveBufferSize=530904
   [junit4]   2> 412182 INFO  (zkCallback-126-thread-1-processing-n:127.0.0.1:35456_gqj) [n:127.0.0.1:35456_gqj    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 413117 INFO  (qtp83303268-703) [n:127.0.0.1:35456_gqj c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 416042 INFO  (qtp83303268-703) [n:127.0.0.1:35456_gqj c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
   [junit4]   2> 417879 INFO  (qtp83303268-703) [n:127.0.0.1:35456_gqj c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 418406 INFO  (qtp83303268-703) [n:127.0.0.1:35456_gqj c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
   [junit4]   2> 418407 INFO  (qtp83303268-703) [n:127.0.0.1:35456_gqj c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@48908242
   [junit4]   2> 418407 INFO  (qtp83303268-703) [n:127.0.0.1:35456_gqj c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 418407 INFO  (qtp83303268-703) [n:127.0.0.1:35456_gqj c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ForceLeaderTest_174C2DA04E69B19C-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ForceLeaderTest_174C2DA04E69B19C-001/control-001/cores/control_collection_shard1_replica_n1/data/]
   [junit4]   2> 418427 INFO  (qtp83303268-703) [n:127.0.0.1:35456_gqj c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=6, maxMergeAtOnceExplicit=7, maxMergedSegmentMB=54.7626953125, floorSegmentMB=0.3525390625, forceMergeDeletesPctAllowed=6.071741754704716, segmentsPerTier=20.0, maxCFSSegmentSizeMB=0.41015625, noCFSRatio=0.0
   [junit4]   2> 418887 WARN  (qtp83303268-703) [n:127.0.0.1:35456_gqj c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 419678 INFO  (qtp83303268-703) [n:127.0.0.1:35456_gqj c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 419678 INFO  (qtp83303268-703) [n:127.0.0.1:35456_gqj c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 419680 INFO  (qtp83303268-703) [n:127.0.0.1:35456_gqj c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 419680 INFO  (qtp83303268-703) [n:127.0.0.1:35456_gqj c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 419722 INFO  (qtp83303268-703) [n:127.0.0.1:35456_gqj c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@717eff23
   [junit4]   2> 419723 INFO  (qtp83303268-703) [n:127.0.0.1:35456_gqj c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@71b0355a[control_collection_shard1_replica_n1] main]
   [junit4]   2> 419738 INFO  (qtp83303268-703) [n:127.0.0.1:35456_gqj c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 419739 INFO  (qtp83303268-703) [n:127.0.0.1:35456_gqj c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 419739 INFO  (qtp83303268-703) [n:127.0.0.1:35456_gqj c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 419741 INFO  (searcherExecutor-287-thread-1-processing-n:127.0.0.1:35456_gqj x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:35456_gqj c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@71b0355a[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 419742 INFO  (qtp83303268-703) [n:127.0.0.1:35456_gqj c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1581252646373687296
   [junit4]   2> 419761 INFO  (qtp83303268-703) [n:127.0.0.1:35456_gqj c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 419761 INFO  (qtp83303268-703) [n:127.0.0.1:35456_gqj c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 419762 INFO  (qtp83303268-703) [n:127.0.0.1:35456_gqj c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:35456/gqj/control_collection_shard1_replica_n1/
   [junit4]   2> 419762 INFO  (qtp83303268-703) [n:127.0.0.1:35456_gqj c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 419762 INFO  (qtp83303268-703) [n:127.0.0.1:35456_gqj c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:35456/gqj/control_collection_shard1_replica_n1/ has no replicas
   [junit4]   2> 419762 INFO  (qtp83303268-703) [n:127.0.0.1:35456_gqj c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 419819 INFO  (zkCallback-126-thread-1-processing-n:127.0.0.1:35456_gqj) [n:127.0.0.1:35456_gqj    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 419820 INFO  (qtp83303268-703) [n:127.0.0.1:35456_gqj c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:35456/gqj/control_collection_shard1_replica_n1/ shard1
   [junit4]   2> 419821 INFO  (qtp83303268-703) [n:127.0.0.1:35456_gqj c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 419874 INFO  (qtp83303268-703) [n:127.0.0.1:35456_gqj c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=7811
   [junit4]   2> 419991 INFO  (zkCallback-126-thread-1-processing-n:127.0.0.1:35456_gqj) [n:127.0.0.1:35456_gqj    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 420402 INFO  (qtp83303268-705) [n:127.0.0.1:35456_gqj    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 420402 INFO  (qtp83303268-705) [n:127.0.0.1:35456_gqj    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:35456_gqj&wt=javabin&version=2} status=0 QTime=8954
   [junit4]   2> 420405 INFO  (OverseerCollectionConfigSetProcessor-98828289239678980-127.0.0.1:35456_gqj-n_0000000000) [n:127.0.0.1:35456_gqj    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 420703 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 420703 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:35064/solr ready
   [junit4]   2> 420705 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 420879 INFO  (SocketProxy-Acceptor-35456) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=52842,localport=35456], receiveBufferSize:531000
   [junit4]   2> 420883 INFO  (SocketProxy-Acceptor-35456) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=41796,localport=57144], receiveBufferSize=530904
   [junit4]   2> 420951 INFO  (qtp83303268-706) [n:127.0.0.1:35456_gqj    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 421018 INFO  (OverseerThreadFactory-284-thread-2-processing-n:127.0.0.1:35456_gqj) [n:127.0.0.1:35456_gqj    ] o.a.s.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 421019 WARN  (OverseerThreadFactory-284-thread-2-processing-n:127.0.0.1:35456_gqj) [n:127.0.0.1:35456_gqj    ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
   [junit4]   2> 421145 INFO  (qtp83303268-706) [n:127.0.0.1:35456_gqj    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 421146 INFO  (qtp83303268-706) [n:127.0.0.1:35456_gqj    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2} status=0 QTime=195
   [junit4]   2> 421147 INFO  (OverseerCollectionConfigSetProcessor-98828289239678980-127.0.0.1:35456_gqj-n_0000000000) [n:127.0.0.1:35456_gqj    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 424044 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ForceLeaderTest_174C2DA04E69B19C-001/shard-1-001 of type TLOG
   [junit4]   2> 424045 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 424111 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1ee05f5d{/gqj,null,AVAILABLE}
   [junit4]   2> 424111 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@2480e4fa{HTTP/1.1,[http/1.1]}{127.0.0.1:44536}
   [junit4]   2> 424111 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.e.j.s.Server Started @444588ms
   [junit4]   2> 424111 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ForceLeaderTest_174C2DA04E69B19C-001/tempDir-001/jetty1, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/gqj, hostPort=46804, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ForceLeaderTest_174C2DA04E69B19C-001/shard-1-001/cores}
   [junit4]   2> 424112 ERROR (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 424112 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 424112 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 424112 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 424181 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-14T16:54:00.966Z
   [junit4]   2> 424490 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 424490 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ForceLeaderTest_174C2DA04E69B19C-001/shard-1-001/solr.xml
   [junit4]   2> 424652 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 424652 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 424653 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@48908242, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 424731 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:35064/solr
   [junit4]   2> 424841 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [n:127.0.0.1:46804_gqj    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 424861 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [n:127.0.0.1:46804_gqj    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 424863 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [n:127.0.0.1:46804_gqj    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:46804_gqj
   [junit4]   2> 425045 INFO  (zkCallback-133-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 425139 INFO  (zkCallback-138-thread-1-processing-n:127.0.0.1:46804_gqj) [n:127.0.0.1:46804_gqj    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 425170 INFO  (zkCallback-126-thread-2-processing-n:127.0.0.1:35456_gqj) [n:127.0.0.1:35456_gqj    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 425855 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [n:127.0.0.1:46804_gqj    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@48908242
   [junit4]   2> 425928 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [n:127.0.0.1:46804_gqj    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@48908242
   [junit4]   2> 425928 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [n:127.0.0.1:46804_gqj    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@48908242
   [junit4]   2> 425929 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [n:127.0.0.1:46804_gqj    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ForceLeaderTest_174C2DA04E69B19C-001/shard-1-001/cores
   [junit4]   2> 426507 INFO  (qtp83303268-707) [n:127.0.0.1:35456_gqj    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:46804_gqj&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 426593 INFO  (OverseerThreadFactory-284-thread-3-processing-n:127.0.0.1:35456_gqj) [n:127.0.0.1:35456_gqj    ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:46804_gqj for creating new replica
   [junit4]   2> 426732 INFO  (SocketProxy-Acceptor-46804) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=44625,localport=46804], receiveBufferSize:531000
   [junit4]   2> 426794 INFO  (SocketProxy-Acceptor-46804) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=44536,localport=39812], receiveBufferSize=530904
   [junit4]   2> 426843 INFO  (qtp1781752430-757) [n:127.0.0.1:46804_gqj    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 426843 INFO  (qtp1781752430-757) [n:127.0.0.1:46804_gqj    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 427049 INFO  (qtp1781752430-757) [n:127.0.0.1:46804_gqj c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 427595 INFO  (qtp1781752430-757) [n:127.0.0.1:46804_gqj c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.s.IndexSchema [collection1_shard2_replica_t41] Schema name=test
   [junit4]   2> 430628 INFO  (qtp1781752430-757) [n:127.0.0.1:46804_gqj c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 430851 INFO  (qtp1781752430-757) [n:127.0.0.1:46804_gqj c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_t41' using configuration from collection collection1, trusted=true
   [junit4]   2> 430852 INFO  (qtp1781752430-757) [n:127.0.0.1:46804_gqj c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_t41' (registry 'solr.core.collection1.shard2.replica_t41') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@48908242
   [junit4]   2> 430852 INFO  (qtp1781752430-757) [n:127.0.0.1:46804_gqj c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 430852 INFO  (qtp1781752430-757) [n:127.0.0.1:46804_gqj c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.SolrCore [[collection1_shard2_replica_t41] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ForceLeaderTest_174C2DA04E69B19C-001/shard-1-001/cores/collection1_shard2_replica_t41], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ForceLeaderTest_174C2DA04E69B19C-001/shard-1-001/cores/collection1_shard2_replica_t41/data/]
   [junit4]   2> 431074 INFO  (qtp1781752430-757) [n:127.0.0.1:46804_gqj c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=6, maxMergeAtOnceExplicit=7, maxMergedSegmentMB=54.7626953125, floorSegmentMB=0.3525390625, forceMergeDeletesPctAllowed=6.071741754704716, segmentsPerTier=20.0, maxCFSSegmentSizeMB=0.41015625, noCFSRatio=0.0
   [junit4]   2> 431076 WARN  (qtp1781752430-757) [n:127.0.0.1:46804_gqj c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 431971 INFO  (qtp1781752430-757) [n:127.0.0.1:46804_gqj c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 431971 INFO  (qtp1781752430-757) [n:127.0.0.1:46804_gqj c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 431972 INFO  (qtp1781752430-757) [n:127.0.0.1:46804_gqj c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 431972 INFO  (qtp1781752430-757) [n:127.0.0.1:46804_gqj c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 432007 INFO  (qtp1781752430-757) [n:127.0.0.1:46804_gqj c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@f88f797
   [junit4]   2> 432007 INFO  (qtp1781752430-757) [n:127.0.0.1:46804_gqj c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.s.SolrIndexSearcher Opening [Searcher@6076dec6[collection1_shard2_replica_t41] main]
   [junit4]   2> 432009 INFO  (qtp1781752430-757) [n:127.0.0.1:46804_gqj c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 432026 INFO  (qtp1781752430-757) [n:127.0.0.1:46804_gqj c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 432047 INFO  (qtp1781752430-757) [n:127.0.0.1:46804_gqj c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 432064 INFO  (searcherExecutor-298-thread-1-processing-n:127.0.0.1:46804_gqj x:collection1_shard2_replica_t41 s:shard2 c:collection1) [n:127.0.0.1:46804_gqj c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.SolrCore [collection1_shard2_replica_t41] Registered new searcher Searcher@6076dec6[collection1_shard2_replica_t41] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 432064 INFO  (qtp1781752430-757) [n:127.0.0.1:46804_gqj c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1581252659294240768
   [junit4]   2> 432116 INFO  (qtp1781752430-757) [n:127.0.0.1:46804_gqj c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 432116 INFO  (qtp1781752430-757) [n:127.0.0.1:46804_gqj c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 432116 INFO  (qtp1781752430-757) [n:127.0.0.1:46804_gqj c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:46804/gqj/collection1_shard2_replica_t41/
   [junit4]   2> 432116 INFO  (qtp1781752430-757) [n:127.0.0.1:46804_gqj c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 432116 INFO  (qtp1781752430-757) [n:127.0.0.1:46804_gqj c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.SyncStrategy http://127.0.0.1:46804/gqj/collection1_shard2_replica_t41/ has no replicas
   [junit4]   2> 432116 INFO  (qtp1781752430-757) [n:127.0.0.1:46804_gqj c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 432116 INFO  (qtp1781752430-757) [n:127.0.0.1:46804_gqj c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.ZkController collection1_shard2_replica_t41 stopping background replication from leader
   [junit4]   2> 432162 INFO  (zkCallback-138-thread-1-processing-n:127.0.0.1:46804_gqj) [n:127.0.0.1:46804_gqj    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 432163 INFO  (qtp1781752430-757) [n:127.0.0.1:46804_gqj c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:46804/gqj/collection1_shard2_replica_t41/ shard2
   [junit4]   2> 432163 INFO  (qtp1781752430-757) [n:127.0.0.1:46804_gqj c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 432182 INFO  (qtp1781752430-757) [n:127.0.0.1:46804_gqj c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=5339
   [junit4]   2> 432203 INFO  (qtp83303268-707) [n:127.0.0.1:35456_gqj    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:46804_gqj&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2} status=0 QTime=5696
   [junit4]   2> 432444 INFO  (zkCallback-138-thread-1-processing-n:127.0.0.1:46804_gqj) [n:127.0.0.1:46804_gqj    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 432955 INFO  (OverseerCollectionConfigSetProcessor-98828289239678980-127.0.0.1:35456_gqj-n_0000000000) [n:127.0.0.1:35456_gqj    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 435644 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ForceLeaderTest_174C2DA04E69B19C-001/shard-2-001 of type TLOG
   [junit4]   2> 435679 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 435699 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4f822ef1{/gqj,null,AVAILABLE}
   [junit4]   2> 435700 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@48b886df{HTTP/1.1,[http/1.1]}{127.0.0.1:45437}
   [junit4]   2> 435700 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.e.j.s.Server Started @456177ms
   [junit4]   2> 435700 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ForceLeaderTest_174C2DA04E69B19C-001/tempDir-001/jetty2, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/gqj, hostPort=50471, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ForceLeaderTest_174C2DA04E69B19C-001/shard-2-001/cores}
   [junit4]   2> 435700 ERROR (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 435763 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 435763 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 435763 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 435763 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-14T16:54:12.617Z
   [junit4]   2> 435967 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 435967 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ForceLeaderTest_174C2DA04E69B19C-001/shard-2-001/solr.xml
   [junit4]   2> 435980 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 435980 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 435982 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[174C2DA04E69B19C]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@48908242, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 436007 INFO  (TEST-ForceLeaderTest.testLas

[...truncated too long message...]

 live nodes from ZooKeeper... (3) -> (2)
   [junit4]   2> 7166122 DEBUG (ScheduledTrigger-4798-thread-1) [n:127.0.0.1:47307_solr    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:34631_solr, 127.0.0.1:47307_solr]
   [junit4]   2> 7166122 DEBUG (ScheduledTrigger-4798-thread-1) [n:127.0.0.1:47307_solr    ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:46772_solr
   [junit4]   2> 7166122 DEBUG (ScheduledTrigger-4798-thread-1) [n:127.0.0.1:47307_solr    ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:52643_solr
   [junit4]   2> 7166122 DEBUG (ScheduledTrigger-4798-thread-1) [n:127.0.0.1:47307_solr    ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:48764_solr
   [junit4]   2> 7166122 DEBUG (ScheduledTrigger-4798-thread-1) [n:127.0.0.1:47307_solr    ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:51808_solr
   [junit4]   2> 7166122 DEBUG (ScheduledTrigger-4798-thread-1) [n:127.0.0.1:47307_solr    ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:51841_solr
   [junit4]   2> 7166130 INFO  (jetty-closer-3323-thread-7) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 7166131 INFO  (jetty-closer-3323-thread-7) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 7166134 DEBUG (ScheduledTrigger-4798-thread-4) [n:127.0.0.1:47307_solr    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 7166134 DEBUG (ScheduledTrigger-4798-thread-4) [n:127.0.0.1:47307_solr    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: [127.0.0.1:34631_solr, 127.0.0.1:47307_solr]
   [junit4]   2> 7166212 DEBUG (jetty-closer-3323-thread-1) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 7167389 WARN  (zkCallback-3426-thread-2-processing-n:127.0.0.1:51808_solr) [n:127.0.0.1:51808_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 7167390 INFO  (jetty-closer-3323-thread-3) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@329b7144{/solr,null,UNAVAILABLE}
   [junit4]   2> 7167411 WARN  (zkCallback-3500-thread-1-processing-n:127.0.0.1:52643_solr) [n:127.0.0.1:52643_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 7167411 INFO  (jetty-closer-3323-thread-5) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@40e830dc{/solr,null,UNAVAILABLE}
   [junit4]   2> 7167610 WARN  (zkCallback-3470-thread-1-processing-n:127.0.0.1:48764_solr) [n:127.0.0.1:48764_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 7167611 INFO  (jetty-closer-3323-thread-4) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@21161308{/solr,null,UNAVAILABLE}
   [junit4]   2> 7167626 WARN  (zkCallback-3416-thread-1-processing-n:127.0.0.1:51841_solr) [n:127.0.0.1:51841_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 7167626 WARN  (zkCallback-3512-thread-1-processing-n:127.0.0.1:46772_solr) [n:127.0.0.1:46772_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 7167627 INFO  (jetty-closer-3323-thread-2) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@256d8bb{/solr,null,UNAVAILABLE}
   [junit4]   2> 7167627 INFO  (jetty-closer-3323-thread-6) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@64d5493c{/solr,null,UNAVAILABLE}
   [junit4]   2> 7167759 WARN  (zkCallback-3374-thread-2-processing-n:127.0.0.1:47307_solr) [n:127.0.0.1:47307_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 7167759 WARN  (zkCallback-3525-thread-1-processing-n:127.0.0.1:34631_solr) [n:127.0.0.1:34631_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 7167760 INFO  (jetty-closer-3323-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@29798334{/solr,null,UNAVAILABLE}
   [junit4]   2> 7167760 INFO  (jetty-closer-3323-thread-7) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@2cf9d44d{/solr,null,UNAVAILABLE}
   [junit4]   2> 7167779 ERROR (SUITE-TriggerIntegrationTest-seed#[174C2DA04E69B19C]-worker) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 7167779 INFO  (SUITE-TriggerIntegrationTest-seed#[174C2DA04E69B19C]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:37628 37628
   [junit4]   2> 7167879 INFO  (Thread-21685) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:37628 37628
   [junit4]   2> 7167890 WARN  (Thread-21685) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	49	/solr/aliases.json
   [junit4]   2> 	14	/solr/security.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	49	/solr/clusterprops.json
   [junit4]   2> 	49	/solr/clusterstate.json
   [junit4]   2> 	5	/solr/autoscaling.json
   [junit4]   2> 	3	/solr/overseer_elect/election/98828724020248632-127.0.0.1:48764_solr-n_0000000009
   [junit4]   2> 	3	/solr/overseer_elect/election/98828724020248598-127.0.0.1:47307_solr-n_0000000004
   [junit4]   2> 	2	/solr/overseer_elect/election/98828724020248617-127.0.0.1:51808_solr-n_0000000007
   [junit4]   2> 	2	/solr/overseer_elect/election/98828724020248613-127.0.0.1:51841_solr-n_0000000006
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	49	/solr/collections
   [junit4]   2> 	42	/solr/live_nodes
   [junit4]   2> 	5	/solr/autoscaling/events/.auto_add_replicas
   [junit4]   2> 	5	/solr/overseer/queue-work
   [junit4]   2> 	5	/solr/overseer/collection-queue-work
   [junit4]   2> 	4	/solr/overseer/queue
   [junit4]   2> 	3	/solr/autoscaling/events/node_added_trigger
   [junit4]   2> 
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.autoscaling.TriggerIntegrationTest_174C2DA04E69B19C-001
   [junit4]   2> Oct 14, 2017 6:46:24 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=FastDecompressionCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=1, maxDocsPerChunk=427, blockSize=815), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=1, blockSize=815)), sim=RandomSimilarity(queryNorm=false): {}, locale=ar-BH, timezone=IST
   [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=108434808,total=509083648
   [junit4]   2> NOTE: All tests run in this JVM: [BlockJoinFacetDistribTest, TestAnalyzedSuggestions, AutoCommitTest, MinimalSchemaTest, PrimitiveFieldTypeTest, SolrInfoBeanTest, SearchHandlerTest, MissingSegmentRecoveryTest, TestRawTransformer, ForceLeaderTest, SortByFunctionTest, LeaderElectionTest, TestSubQueryTransformerDistrib, TestQuerySenderListener, TestHttpShardHandlerFactory, TestNumericRangeQuery32, HdfsLockFactoryTest, ClusterStateTest, CollectionStateFormat2Test, BasicAuthIntegrationTest, PreAnalyzedFieldTest, SolrTestCaseJ4Test, AssignBackwardCompatibilityTest, TestJmxIntegration, TestRestoreCore, DistributedQueueTest, URLClassifyProcessorTest, ChaosMonkeyNothingIsSafeWithPullReplicasTest, TestApiFramework, TestSuggestSpellingConverter, UniqFieldsUpdateProcessorFactoryTest, SolrMetricsIntegrationTest, TestConfigSetImmutable, TestPayloadScoreQParserPlugin, HardAutoCommitTest, TestOrdValues, HdfsChaosMonkeyNothingIsSafeTest, TestPseudoReturnFields, HdfsTlogReplayBufferedWhileIndexingTest, OverseerTest, TestDownShardTolerantSearch, TestExactSharedStatsCache, TestConfigSetsAPI, SignatureUpdateProcessorFactoryTest, CheckHdfsIndexTest, FullSolrCloudDistribCmdsTest, CloudMLTQParserTest, DistributedQueryComponentCustomSortTest, ZkFailoverTest, TestJavabinTupleStreamParser, HdfsAutoAddReplicasIntegrationTest, TestMacroExpander, IgnoreCommitOptimizeUpdateProcessorFactoryTest, CleanupOldIndexTest, TestReload, TestManagedSchemaThreadSafety, TestJsonFacetRefinement, TestDocBasedVersionConstraints, CachingDirectoryFactoryTest, TestUtils, TestReversedWildcardFilterFactory, ReturnFieldsTest, TestRequestStatusCollectionAPI, CdcrReplicationHandlerTest, TestLeaderElectionWithEmptyReplica, ConnectionManagerTest, RestartWhileUpdatingTest, TestSchemalessBufferedUpdates, TestZkChroot, TestCloudManagedSchema, TestHdfsBackupRestoreCore, StressHdfsTest, ShufflingReplicaListTransformerTest, TestCryptoKeys, TestAnalyzeInfixSuggestions, TestRecovery, TestFieldCacheVsDocValues, DistribCursorPagingTest, QueryEqualityTest, PingRequestHandlerTest, SmileWriterTest, DateFieldTest, TestCloudInspectUtil, TestEmbeddedSolrServerConstructors, TestClusterProperties, AutoScalingHandlerTest, TriggerIntegrationTest]
   [junit4] Completed [668/745 (7!)] on J2 in 131.94s, 10 tests, 1 failure <<< FAILURES!

[...truncated 47174 lines...]