You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2018/03/15 11:26:57 UTC

[JENKINS] Lucene-Solr-7.3-Windows (32bit/jdk1.8.0_144) - Build # 4 - Unstable!

Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.3-Windows/4/
Java: 32bit/jdk1.8.0_144 -server -XX:+UseParallelGC

9 tests failed.
FAILED:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:55697/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:55697/collection1
	at __randomizedtesting.SeedInfo.seed([B81645FECBDB01CB:30427A2465276C33]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:654)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1105)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:885)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:818)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:484)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:463)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.commit(AbstractFullDistribZkTestBase.java:1591)
	at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:212)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:171)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:542)
	... 53 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest

Error Message:
4 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest:     1) Thread[id=113, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=115, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)    3) Thread[id=116, name=zkConnectionManagerCallback-25-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=114, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]-SendThread(127.0.0.1:55601), state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method)         at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296)         at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278)         at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1144)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 4 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest: 
   1) Thread[id=113, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=115, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)
   3) Thread[id=116, name=zkConnectionManagerCallback-25-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=114, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]-SendThread(127.0.0.1:55601), state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method)
        at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296)
        at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278)
        at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1144)
	at __randomizedtesting.SeedInfo.seed([B81645FECBDB01CB]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=116, name=zkConnectionManagerCallback-25-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=114, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]-SendThread(127.0.0.1:55601), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=116, name=zkConnectionManagerCallback-25-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=114, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]-SendThread(127.0.0.1:55601), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
	at __randomizedtesting.SeedInfo.seed([B81645FECBDB01CB]:0)


FAILED:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:55715/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:55715/collection1
	at __randomizedtesting.SeedInfo.seed([B81645FECBDB01CB:30427A2465276C33]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:654)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1105)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:885)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:818)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:484)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:463)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.commit(AbstractFullDistribZkTestBase.java:1591)
	at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:212)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:171)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:542)
	... 53 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest

Error Message:
4 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest:     1) Thread[id=115, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)    2) Thread[id=114, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]-SendThread(127.0.0.1:55610), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:230)         at org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1249)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1173)    3) Thread[id=113, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=116, name=zkConnectionManagerCallback-25-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 4 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest: 
   1) Thread[id=115, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)
   2) Thread[id=114, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]-SendThread(127.0.0.1:55610), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:230)
        at org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1249)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1173)
   3) Thread[id=113, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=116, name=zkConnectionManagerCallback-25-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([B81645FECBDB01CB]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=114, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]-SendThread(127.0.0.1:55610), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1054)    2) Thread[id=116, name=zkConnectionManagerCallback-25-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=114, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]-SendThread(127.0.0.1:55610), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1054)
   2) Thread[id=116, name=zkConnectionManagerCallback-25-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([B81645FECBDB01CB]:0)


FAILED:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:55515/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:55515/collection1
	at __randomizedtesting.SeedInfo.seed([B81645FECBDB01CB:30427A2465276C33]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:654)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1105)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:885)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:818)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:484)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:463)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.commit(AbstractFullDistribZkTestBase.java:1591)
	at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:212)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:171)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:542)
	... 53 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest

Error Message:
4 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest:     1) Thread[id=1357, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)    2) Thread[id=1355, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=1358, name=zkConnectionManagerCallback-248-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=1356, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]-SendThread(127.0.0.1:55442), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 4 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest: 
   1) Thread[id=1357, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)
   2) Thread[id=1355, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=1358, name=zkConnectionManagerCallback-248-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=1356, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]-SendThread(127.0.0.1:55442), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
	at __randomizedtesting.SeedInfo.seed([B81645FECBDB01CB]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=1358, name=zkConnectionManagerCallback-248-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=1356, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]-SendThread(127.0.0.1:55442), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=1358, name=zkConnectionManagerCallback-248-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=1356, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]-SendThread(127.0.0.1:55442), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
	at __randomizedtesting.SeedInfo.seed([B81645FECBDB01CB]:0)




Build Log:
[...truncated 12303 lines...]
   [junit4] Suite: org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest
   [junit4]   2> Creating dataDir: C:\Users\jenkins\workspace\Lucene-Solr-7.3-Windows\solr\build\solr-core\test\J1\temp\solr.cloud.ChaosMonkeyNothingIsSafeTest_B81645FECBDB01CB-001\init-core-data-001
   [junit4]   2> 149360 WARN  (SUITE-ChaosMonkeyNothingIsSafeTest-seed#[B81645FECBDB01CB]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=21 numCloses=21
   [junit4]   2> 149360 INFO  (SUITE-ChaosMonkeyNothingIsSafeTest-seed#[B81645FECBDB01CB]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 149366 INFO  (SUITE-ChaosMonkeyNothingIsSafeTest-seed#[B81645FECBDB01CB]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
   [junit4]   2> 149366 INFO  (SUITE-ChaosMonkeyNothingIsSafeTest-seed#[B81645FECBDB01CB]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 149366 INFO  (SUITE-ChaosMonkeyNothingIsSafeTest-seed#[B81645FECBDB01CB]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
   [junit4]   2> 149370 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 149370 INFO  (Thread-186) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 149370 INFO  (Thread-186) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 149375 ERROR (Thread-186) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 149472 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.a.s.c.ZkTestServer start zk server on port:55442
   [junit4]   2> 149477 INFO  (zkConnectionManagerCallback-225-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 149485 INFO  (zkConnectionManagerCallback-227-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 149493 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.a.s.c.AbstractZkTestCase put C:\Users\jenkins\workspace\Lucene-Solr-7.3-Windows\solr\core\src\test-files\solr\collection1\conf\solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 149495 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.a.s.c.AbstractZkTestCase put C:\Users\jenkins\workspace\Lucene-Solr-7.3-Windows\solr\core\src\test-files\solr\collection1\conf\schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 149497 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.a.s.c.AbstractZkTestCase put C:\Users\jenkins\workspace\Lucene-Solr-7.3-Windows\solr\core\src\test-files\solr\collection1\conf\solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 149498 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.a.s.c.AbstractZkTestCase put C:\Users\jenkins\workspace\Lucene-Solr-7.3-Windows\solr\core\src\test-files\solr\collection1\conf\stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 149500 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.a.s.c.AbstractZkTestCase put C:\Users\jenkins\workspace\Lucene-Solr-7.3-Windows\solr\core\src\test-files\solr\collection1\conf\protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 149501 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.a.s.c.AbstractZkTestCase put C:\Users\jenkins\workspace\Lucene-Solr-7.3-Windows\solr\core\src\test-files\solr\collection1\conf\currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 149502 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.a.s.c.AbstractZkTestCase put C:\Users\jenkins\workspace\Lucene-Solr-7.3-Windows\solr\core\src\test-files\solr\collection1\conf\enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 149504 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.a.s.c.AbstractZkTestCase put C:\Users\jenkins\workspace\Lucene-Solr-7.3-Windows\solr\core\src\test-files\solr\collection1\conf\open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 149508 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.a.s.c.AbstractZkTestCase put C:\Users\jenkins\workspace\Lucene-Solr-7.3-Windows\solr\core\src\test-files\solr\collection1\conf\mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 149510 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.a.s.c.AbstractZkTestCase put C:\Users\jenkins\workspace\Lucene-Solr-7.3-Windows\solr\core\src\test-files\solr\collection1\conf\old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 149512 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.a.s.c.AbstractZkTestCase put C:\Users\jenkins\workspace\Lucene-Solr-7.3-Windows\solr\core\src\test-files\solr\collection1\conf\synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 149514 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x1000522d7470001, likely client has closed socket
   [junit4]   2> 149514 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Will use TLOG replicas unless explicitly asked otherwise
   [junit4]   2> 150043 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T04:27:37+07:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 150044 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 150044 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 150044 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.e.j.s.session Scavenging every 600000ms
   [junit4]   2> 150044 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@783905{/,null,AVAILABLE}
   [junit4]   2> 150049 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@1ec98e5{HTTP/1.1,[http/1.1]}{127.0.0.1:55451}
   [junit4]   2> 150049 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.e.j.s.Server Started @161601ms
   [junit4]   2> 150049 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=C:\Users\jenkins\workspace\Lucene-Solr-7.3-Windows\solr\build\solr-core\test\J1\temp\solr.cloud.ChaosMonkeyNothingIsSafeTest_B81645FECBDB01CB-001\tempDir-001/control/data, replicaType=NRT, hostContext=/, hostPort=55451, coreRootDirectory=C:\Users\jenkins\workspace\Lucene-Solr-7.3-Windows\solr\build\solr-core\test\J1\..\..\..\..\..\..\..\..\..\Users\jenkins\workspace\Lucene-Solr-7.3-Windows\solr\build\solr-core\test\J1\temp\solr.cloud.ChaosMonkeyNothingIsSafeTest_B81645FECBDB01CB-001\control-001\cores}
   [junit4]   2> 150049 ERROR (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 150049 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.3.0
   [junit4]   2> 150049 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 150049 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 150049 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-03-15T10:27:00.174Z
   [junit4]   2> 150052 INFO  (zkConnectionManagerCallback-229-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 150054 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 150054 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from C:\Users\jenkins\workspace\Lucene-Solr-7.3-Windows\solr\build\solr-core\test\J1\temp\solr.cloud.ChaosMonkeyNothingIsSafeTest_B81645FECBDB01CB-001\control-001\solr.xml
   [junit4]   2> 150061 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 150061 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 150061 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@3462c8, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 150066 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:55442/solr
   [junit4]   2> 150069 INFO  (zkConnectionManagerCallback-233-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 150073 INFO  (zkConnectionManagerCallback-235-thread-1-processing-n:127.0.0.1:55451_) [n:127.0.0.1:55451_    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 150180 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [n:127.0.0.1:55451_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 150181 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [n:127.0.0.1:55451_    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:55451_
   [junit4]   2> 150181 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [n:127.0.0.1:55451_    ] o.a.s.c.Overseer Overseer (id=72063241236709380-127.0.0.1:55451_-n_0000000000) starting
   [junit4]   2> 150190 INFO  (zkConnectionManagerCallback-240-thread-1-processing-n:127.0.0.1:55451_) [n:127.0.0.1:55451_    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 150192 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [n:127.0.0.1:55451_    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:55442/solr ready
   [junit4]   2> 150193 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [n:127.0.0.1:55451_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:55451_
   [junit4]   2> 150195 INFO  (OverseerStateUpdate-72063241236709380-127.0.0.1:55451_-n_0000000000) [n:127.0.0.1:55451_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 150236 INFO  (zkCallback-239-thread-1-processing-n:127.0.0.1:55451_) [n:127.0.0.1:55451_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 150368 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [n:127.0.0.1:55451_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3462c8
   [junit4]   2> 150384 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [n:127.0.0.1:55451_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3462c8
   [junit4]   2> 150384 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [n:127.0.0.1:55451_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3462c8
   [junit4]   2> 150387 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [n:127.0.0.1:55451_    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath C:\Users\jenkins\workspace\Lucene-Solr-7.3-Windows\solr\build\solr-core\test\J1\..\..\..\..\..\..\..\..\..\Users\jenkins\workspace\Lucene-Solr-7.3-Windows\solr\build\solr-core\test\J1\temp\solr.cloud.ChaosMonkeyNothingIsSafeTest_B81645FECBDB01CB-001\control-001\cores
   [junit4]   2> 150452 INFO  (zkConnectionManagerCallback-244-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 150453 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 150454 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:55442/solr ready
   [junit4]   2> 150459 INFO  (qtp25935756-1309) [n:127.0.0.1:55451_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:55451_&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 150461 INFO  (OverseerThreadFactory-754-thread-1-processing-n:127.0.0.1:55451_) [n:127.0.0.1:55451_    ] o.a.s.c.a.c.CreateCollectionCmd Create collection control_collection
   [junit4]   2> 150572 INFO  (qtp25935756-1312) [n:127.0.0.1:55451_    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 150574 INFO  (qtp25935756-1312) [n:127.0.0.1:55451_    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 150681 INFO  (zkCallback-234-thread-1-processing-n:127.0.0.1:55451_) [n:127.0.0.1:55451_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 151601 INFO  (qtp25935756-1312) [n:127.0.0.1:55451_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
   [junit4]   2> 151613 INFO  (qtp25935756-1312) [n:127.0.0.1:55451_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
   [junit4]   2> 151658 INFO  (qtp25935756-1312) [n:127.0.0.1:55451_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 151679 INFO  (qtp25935756-1312) [n:127.0.0.1:55451_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
   [junit4]   2> 151679 INFO  (qtp25935756-1312) [n:127.0.0.1:55451_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3462c8
   [junit4]   2> 151700 INFO  (qtp25935756-1312) [n:127.0.0.1:55451_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 151700 INFO  (qtp25935756-1312) [n:127.0.0.1:55451_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [C:\Users\jenkins\workspace\Lucene-Solr-7.3-Windows\solr\build\solr-core\test\J1\temp\solr.cloud.ChaosMonkeyNothingIsSafeTest_B81645FECBDB01CB-001\control-001\cores\control_collection_shard1_replica_n1], dataDir=[C:\Users\jenkins\workspace\Lucene-Solr-7.3-Windows\solr\build\solr-core\test\J1\..\..\..\..\..\..\..\..\..\Users\jenkins\workspace\Lucene-Solr-7.3-Windows\solr\build\solr-core\test\J1\temp\solr.cloud.ChaosMonkeyNothingIsSafeTest_B81645FECBDB01CB-001\control-001\cores\control_collection_shard1_replica_n1\data\]
   [junit4]   2> 151706 INFO  (qtp25935756-1312) [n:127.0.0.1:55451_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=43, maxMergeAtOnceExplicit=26, maxMergedSegmentMB=79.244140625, floorSegmentMB=1.2578125, forceMergeDeletesPctAllowed=16.107352262440177, segmentsPerTier=40.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 151796 WARN  (qtp25935756-1312) [n:127.0.0.1:55451_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 151897 INFO  (qtp25935756-1312) [n:127.0.0.1:55451_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 151897 INFO  (qtp25935756-1312) [n:127.0.0.1:55451_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 151899 INFO  (qtp25935756-1312) [n:127.0.0.1:55451_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms; 
   [junit4]   2> 151899 INFO  (qtp25935756-1312) [n:127.0.0.1:55451_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 151900 INFO  (qtp25935756-1312) [n:127.0.0.1:55451_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=11, maxMergeAtOnceExplicit=25, maxMergedSegmentMB=29.64453125, floorSegmentMB=1.9287109375, forceMergeDeletesPctAllowed=14.235018749246727, segmentsPerTier=20.0, maxCFSSegmentSizeMB=0.2783203125, noCFSRatio=0.0
   [junit4]   2> 151900 INFO  (qtp25935756-1312) [n:127.0.0.1:55451_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@f42072[control_collection_shard1_replica_n1] main]
   [junit4]   2> 151902 INFO  (qtp25935756-1312) [n:127.0.0.1:55451_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 151902 INFO  (qtp25935756-1312) [n:127.0.0.1:55451_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 151903 INFO  (qtp25935756-1312) [n:127.0.0.1:55451_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 151903 INFO  (qtp25935756-1312) [n:127.0.0.1:55451_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1594999043027632128
   [junit4]   2> 151905 INFO  (searcherExecutor-757-thread-1-processing-n:127.0.0.1:55451_ x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:55451_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@f42072[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 151908 INFO  (qtp25935756-1312) [n:127.0.0.1:55451_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkShardTerms Successful update terms at /collections/control_collection/terms/shard1 to Terms{values={core_node2=0}, version=0}
   [junit4]   2> 151911 INFO  (qtp25935756-1312) [n:127.0.0.1:55451_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 151911 INFO  (qtp25935756-1312) [n:127.0.0.1:55451_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 151911 INFO  (qtp25935756-1312) [n:127.0.0.1:55451_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:55451/control_collection_shard1_replica_n1/
   [junit4]   2> 151911 INFO  (qtp25935756-1312) [n:127.0.0.1:55451_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 151911 INFO  (qtp25935756-1312) [n:127.0.0.1:55451_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:55451/control_collection_shard1_replica_n1/ has no replicas
   [junit4]   2> 151911 INFO  (qtp25935756-1312) [n:127.0.0.1:55451_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 151914 INFO  (qtp25935756-1312) [n:127.0.0.1:55451_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:55451/control_collection_shard1_replica_n1/ shard1
   [junit4]   2> 152017 INFO  (zkCallback-234-thread-2-processing-n:127.0.0.1:55451_) [n:127.0.0.1:55451_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 152067 INFO  (qtp25935756-1312) [n:127.0.0.1:55451_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 152072 INFO  (qtp25935756-1312) [n:127.0.0.1:55451_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1499
   [junit4]   2> 152076 INFO  (qtp25935756-1309) [n:127.0.0.1:55451_    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 152174 INFO  (zkCallback-234-thread-2-processing-n:127.0.0.1:55451_) [n:127.0.0.1:55451_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 152466 INFO  (OverseerCollectionConfigSetProcessor-72063241236709380-127.0.0.1:55451_-n_0000000000) [n:127.0.0.1:55451_    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 153077 INFO  (qtp25935756-1309) [n:127.0.0.1:55451_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:55451_&wt=javabin&version=2} status=0 QTime=2618
   [junit4]   2> 153078 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x1000522d7470006, likely client has closed socket
   [junit4]   2> 153086 INFO  (zkConnectionManagerCallback-248-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 153087 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 153088 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:55442/solr ready
   [junit4]   2> 153088 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 153090 INFO  (qtp25935756-1312) [n:127.0.0.1:55451_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 153092 INFO  (OverseerThreadFactory-754-thread-2-processing-n:127.0.0.1:55451_) [n:127.0.0.1:55451_    ] o.a.s.c.a.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 153093 WARN  (OverseerThreadFactory-754-thread-2-processing-n:127.0.0.1:55451_) [n:127.0.0.1:55451_    ] o.a.s.c.a.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
   [junit4]   2> 153299 INFO  (qtp25935756-1312) [n:127.0.0.1:55451_    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 153299 INFO  (qtp25935756-1312) [n:127.0.0.1:55451_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=208
   [junit4]   2> 153836 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory C:\Users\jenkins\workspace\Lucene-Solr-7.3-Windows\solr\build\solr-core\test\J1\temp\solr.cloud.ChaosMonkeyNothingIsSafeTest_B81645FECBDB01CB-001\shard-1-001 of type TLOG
   [junit4]   2> 153836 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T04:27:37+07:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 153837 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 153837 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 153837 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.e.j.s.session Scavenging every 660000ms
   [junit4]   2> 153837 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@486d65{/,null,AVAILABLE}
   [junit4]   2> 153838 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@16ce2b6{HTTP/1.1,[http/1.1]}{127.0.0.1:55491}
   [junit4]   2> 153839 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.e.j.s.Server Started @165390ms
   [junit4]   2> 153839 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=C:\Users\jenkins\workspace\Lucene-Solr-7.3-Windows\solr\build\solr-core\test\J1\temp\solr.cloud.ChaosMonkeyNothingIsSafeTest_B81645FECBDB01CB-001\tempDir-001/jetty1, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/, hostPort=55491, coreRootDirectory=C:\Users\jenkins\workspace\Lucene-Solr-7.3-Windows\solr\build\solr-core\test\J1\..\..\..\..\..\..\..\..\..\Users\jenkins\workspace\Lucene-Solr-7.3-Windows\solr\build\solr-core\test\J1\temp\solr.cloud.ChaosMonkeyNothingIsSafeTest_B81645FECBDB01CB-001\shard-1-001\cores}
   [junit4]   2> 153839 ERROR (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 153839 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.3.0
   [junit4]   2> 153839 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 153839 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 153839 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-03-15T10:27:03.964Z
   [junit4]   2> 153843 INFO  (zkConnectionManagerCallback-250-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 153844 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x1000522d7470008, likely client has closed socket
   [junit4]   2> 153845 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 153845 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from C:\Users\jenkins\workspace\Lucene-Solr-7.3-Windows\solr\build\solr-core\test\J1\temp\solr.cloud.ChaosMonkeyNothingIsSafeTest_B81645FECBDB01CB-001\shard-1-001\solr.xml
   [junit4]   2> 153851 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 153851 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 153852 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@3462c8, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 153858 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:55442/solr
   [junit4]   2> 153861 INFO  (zkConnectionManagerCallback-254-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 153863 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x1000522d7470009, likely client has closed socket
   [junit4]   2> 153866 INFO  (zkConnectionManagerCallback-256-thread-1-processing-n:127.0.0.1:55491_) [n:127.0.0.1:55491_    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 153874 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [n:127.0.0.1:55491_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 153877 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [n:127.0.0.1:55491_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 153879 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [n:127.0.0.1:55491_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:55491_
   [junit4]   2> 153881 INFO  (zkCallback-234-thread-1-processing-n:127.0.0.1:55451_) [n:127.0.0.1:55451_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 153881 INFO  (zkCallback-239-thread-1-processing-n:127.0.0.1:55451_) [n:127.0.0.1:55451_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 153882 INFO  (zkCallback-247-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 153883 INFO  (zkCallback-255-thread-1-processing-n:127.0.0.1:55491_) [n:127.0.0.1:55491_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 153975 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [n:127.0.0.1:55491_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3462c8
   [junit4]   2> 153981 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [n:127.0.0.1:55491_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3462c8
   [junit4]   2> 153981 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [n:127.0.0.1:55491_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3462c8
   [junit4]   2> 153984 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [n:127.0.0.1:55491_    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath C:\Users\jenkins\workspace\Lucene-Solr-7.3-Windows\solr\build\solr-core\test\J1\..\..\..\..\..\..\..\..\..\Users\jenkins\workspace\Lucene-Solr-7.3-Windows\solr\build\solr-core\test\J1\temp\solr.cloud.ChaosMonkeyNothingIsSafeTest_B81645FECBDB01CB-001\shard-1-001\cores
   [junit4]   2> 153990 INFO  (zkConnectionManagerCallback-261-thread-1-processing-n:127.0.0.1:55491_) [n:127.0.0.1:55491_    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 153992 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [n:127.0.0.1:55491_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 153993 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]) [n:127.0.0.1:55491_    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:55442/solr ready
   [junit4]   2> 154034 INFO  (qtp25935756-1308) [n:127.0.0.1:55451_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:55491_&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 154037 INFO  (OverseerCollectionConfigSetProcessor-72063241236709380-127.0.0.1:55451_-n_0000000000) [n:127.0.0.1:55451_    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 154038 INFO  (OverseerThreadFactory-754-thread-3-processing-n:127.0.0.1:55451_) [n:127.0.0.1:55451_    ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:55491_ for creating new replica
   [junit4]   2> 154046 INFO  (qtp5420874-1361) [n:127.0.0.1:55491_    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t21&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 154046 INFO  (qtp5420874-1361) [n:127.0.0.1:55491_    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 155079 INFO  (qtp5420874-1361) [n:127.0.0.1:55491_ c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
   [junit4]   2> 155087 INFO  (qtp5420874-1361) [n:127.0.0.1:55491_ c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.s.IndexSchema [collection1_shard1_replica_t21] Schema name=test
   [junit4]   2> 155135 INFO  (qtp5420874-1361) [n:127.0.0.1:55491_ c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 155158 INFO  (qtp5420874-1361) [n:127.0.0.1:55491_ c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_t21' using configuration from collection collection1, trusted=true
   [junit4]   2> 155158 INFO  (qtp5420874-1361) [n:127.0.0.1:55491_ c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_t21' (registry 'solr.core.collection1.shard1.replica_t21') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3462c8
   [junit4]   2> 155158 INFO  (qtp5420874-1361) [n:127.0.0.1:55491_ c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 155158 INFO  (qtp5420874-1361) [n:127.0.0.1:55491_ c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.SolrCore [[collection1_shard1_replica_t21] ] Opening new SolrCore at [C:\Users\jenkins\workspace\Lucene-Solr-7.3-Windows\solr\build\solr-core\test\J1\temp\solr.cloud.ChaosMonkeyNothingIsSafeTest_B81645FECBDB01CB-001\shard-1-001\cores\collection1_shard1_replica_t21], dataDir=[C:\Users\jenkins\workspace\Lucene-Solr-7.3-Windows\solr\build\solr-core\test\J1\..\..\..\..\..\..\..\..\..\Users\jenkins\workspace\Lucene-Solr-7.3-Windows\solr\build\solr-core\test\J1\temp\solr.cloud.ChaosMonkeyNothingIsSafeTest_B81645FECBDB01CB-001\shard-1-001\cores\collection1_shard1_replica_t21\data\]
   [junit4]   2> 155167 INFO  (qtp5420874-1361) [n:127.0.0.1:55491_ c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=43, maxMergeAtOnceExplicit=26, maxMergedSegmentMB=79.244140625, floorSegmentMB=1.2578125, forceMergeDeletesPctAllowed=16.107352262440177, segmentsPerTier=40.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 155196 WARN  (qtp5420874-1361) [n:127.0.0.1:55491_ c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 155268 INFO  (qtp5420874-1361) [n:127.0.0.1:55491_ c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 155268 INFO  (qtp5420874-1361) [n:127.0.0.1:55491_ c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 155269 INFO  (qtp5420874-1361) [n:127.0.0.1:55491_ c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms; 
   [junit4]   2> 155269 INFO  (qtp5420874-1361) [n:127.0.0.1:55491_ c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 155270 INFO  (qtp5420874-1361) [n:127.0.0.1:55491_ c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=11, maxMergeAtOnceExplicit=25, maxMergedSegmentMB=29.64453125, floorSegmentMB=1.9287109375, forceMergeDeletesPctAllowed=14.235018749246727, segmentsPerTier=20.0, maxCFSSegmentSizeMB=0.2783203125, noCFSRatio=0.0
   [junit4]   2> 155270 INFO  (qtp5420874-1361) [n:127.0.0.1:55491_ c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.s.SolrIndexSearcher Opening [Searcher@3f35c4[collection1_shard1_replica_t21] main]
   [junit4]   2> 155272 INFO  (qtp5420874-1361) [n:127.0.0.1:55491_ c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 155273 INFO  (qtp5420874-1361) [n:127.0.0.1:55491_ c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 155273 INFO  (qtp5420874-1361) [n:127.0.0.1:55491_ c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 155274 INFO  (searcherExecutor-768-thread-1-processing-n:127.0.0.1:55491_ x:collection1_shard1_replica_t21 s:shard1 c:collection1) [n:127.0.0.1:55491_ c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.SolrCore [collection1_shard1_replica_t21] Registered new searcher Searcher@3f35c4[collection1_shard1_replica_t21] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 155274 INFO  (qtp5420874-1361) [n:127.0.0.1:55491_ c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1594999046562381824
   [junit4]   2> 155279 INFO  (qtp5420874-1361) [n:127.0.0.1:55491_ c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.ZkShardTerms Successful update terms at /collections/collection1/terms/shard1 to Terms{values={core_node22=0}, version=0}
   [junit4]   2> 155281 INFO  (qtp5420874-1361) [n:127.0.0.1:55491_ c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 155281 INFO  (qtp5420874-1361) [n:127.0.0.1:55491_ c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 155281 INFO  (qtp5420874-1361) [n:127.0.0.1:55491_ c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:55491/collection1_shard1_replica_t21/
   [junit4]   2> 155281 INFO  (qtp5420874-1361) [n:127.0.0.1:55491_ c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 155281 INFO  (qtp5420874-1361) [n:127.0.0.1:55491_ c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.SyncStrategy http://127.0.0.1:55491/collection1_shard1_replica_t21/ has no 

[...truncated too long message...]

d:
   [junit4]   2>    1) Thread[id=116, name=zkConnectionManagerCallback-25-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]   2>         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2>    2) Thread[id=114, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]-SendThread(127.0.0.1:55601), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
   [junit4]   2>         at java.lang.Thread.sleep(Native Method)
   [junit4]   2>         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)
   [junit4]   2>         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)
   [junit4]   2>         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {rnd_b=PostingsFormat(name=Memory), a_t=PostingsFormat(name=LuceneVarGapDocFreqInterval), id=PostingsFormat(name=Memory)}, docValues:{_version_=DocValuesFormat(name=Asserting)}, maxPointsInLeafNode=652, maxMBSortInHeap=5.506049091702862, sim=RandomSimilarity(queryNorm=false): {}, locale=it-CH, timezone=Africa/Kampala
   [junit4]   2> NOTE: Windows 10 10.0 x86/Oracle Corporation 1.8.0_144 (32-bit)/cpus=3,threads=3,free=86603704,total=239075328
   [junit4]   2> NOTE: All tests run in this JVM: [ChaosMonkeyNothingIsSafeTest]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ChaosMonkeyNothingIsSafeTest -Dtests.seed=B81645FECBDB01CB -Dtests.slow=true -Dtests.badapples=true -Dtests.locale=it-CH -Dtests.timezone=Africa/Kampala -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   0.00s J1 | ChaosMonkeyNothingIsSafeTest (suite) <<<
   [junit4]    > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 4 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest: 
   [junit4]    >    1) Thread[id=113, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    2) Thread[id=115, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)
   [junit4]    >    3) Thread[id=116, name=zkConnectionManagerCallback-25-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    4) Thread[id=114, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]-SendThread(127.0.0.1:55601), state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]
   [junit4]    >         at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method)
   [junit4]    >         at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296)
   [junit4]    >         at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278)
   [junit4]    >         at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159)
   [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]    >         at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1144)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([B81645FECBDB01CB]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   [junit4]    >    1) Thread[id=116, name=zkConnectionManagerCallback-25-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    2) Thread[id=114, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B81645FECBDB01CB]-SendThread(127.0.0.1:55601), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([B81645FECBDB01CB]:0)
   [junit4] Completed [5/5 (2!)] on J1 in 160.10s, 1 test, 3 errors <<< FAILURES!

[...truncated 13 lines...]
BUILD FAILED
C:\Users\jenkins\workspace\Lucene-Solr-7.3-Windows\lucene\common-build.xml:1561: The following error occurred while executing this line:
C:\Users\jenkins\workspace\Lucene-Solr-7.3-Windows\lucene\common-build.xml:1085: There were test failures: 5 suites, 5 tests, 4 suite-level errors, 2 errors, 3 ignored [seed: B81645FECBDB01CB]

Total time: 2 minutes 44 seconds

[repro] Setting last failure code to 1

[repro] Failures:
[repro]   2/5 failed: org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest
[repro] Exiting with code 1

C:\Users\jenkins\workspace\Lucene-Solr-7.3-Windows>IF EXIST lucene\build move /y lucene\build lucene\build.repro 
        1 dir(s) moved.

C:\Users\jenkins\workspace\Lucene-Solr-7.3-Windows>IF EXIST solr\build move /y solr\build solr\build.repro 
        1 dir(s) moved.

C:\Users\jenkins\workspace\Lucene-Solr-7.3-Windows>IF EXIST lucene\build.orig move /y lucene\build.orig lucene\build 
        1 dir(s) moved.

C:\Users\jenkins\workspace\Lucene-Solr-7.3-Windows>IF EXIST solr\build.orig move /y solr\build.orig solr\build 
        1 dir(s) moved.

C:\Users\jenkins\workspace\Lucene-Solr-7.3-Windows>exit 0 
Archiving artifacts
Setting ANT_1_8_2_HOME=C:\Users\jenkins\tools\hudson.tasks.Ant_AntInstallation\ANT_1.8.2
[WARNINGS] Parsing warnings in console log with parser Java Compiler (javac)
Setting ANT_1_8_2_HOME=C:\Users\jenkins\tools\hudson.tasks.Ant_AntInstallation\ANT_1.8.2
Setting ANT_1_8_2_HOME=C:\Users\jenkins\tools\hudson.tasks.Ant_AntInstallation\ANT_1.8.2
<Git Blamer> Using GitBlamer to create author and commit information for all warnings.
<Git Blamer> GIT_COMMIT=24c8ae59548f887f303edbae7d74598140d6fcd2, workspace=C:\Users\jenkins\workspace\Lucene-Solr-7.3-Windows
[WARNINGS] Computing warning deltas based on reference build #2
Recording test results
Setting ANT_1_8_2_HOME=C:\Users\jenkins\tools\hudson.tasks.Ant_AntInstallation\ANT_1.8.2
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)
Setting ANT_1_8_2_HOME=C:\Users\jenkins\tools\hudson.tasks.Ant_AntInstallation\ANT_1.8.2
Setting ANT_1_8_2_HOME=C:\Users\jenkins\tools\hudson.tasks.Ant_AntInstallation\ANT_1.8.2
Setting ANT_1_8_2_HOME=C:\Users\jenkins\tools\hudson.tasks.Ant_AntInstallation\ANT_1.8.2
Setting ANT_1_8_2_HOME=C:\Users\jenkins\tools\hudson.tasks.Ant_AntInstallation\ANT_1.8.2

[JENKINS] Lucene-Solr-7.3-Windows (64bit/jdk-9.0.1) - Build # 5 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.3-Windows/5/
Java: 64bit/jdk-9.0.1 -XX:+UseCompressedOops -XX:+UseSerialGC

1 tests failed.
FAILED:  org.apache.solr.core.ExitableDirectoryReaderTest.testQueryResults

Error Message:
Should NOT have inserted partial results! expected:<4> but was:<5>

Stack Trace:
java.lang.AssertionError: Should NOT have inserted partial results! expected:<4> but was:<5>
	at __randomizedtesting.SeedInfo.seed([193A467CB4A58EEB:44B37E0516AED0A0]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.apache.solr.core.ExitableDirectoryReaderTest.testQueryResults(ExitableDirectoryReaderTest.java:141)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.base/java.lang.Thread.run(Thread.java:844)




Build Log:
[...truncated 14426 lines...]
   [junit4] Suite: org.apache.solr.core.ExitableDirectoryReaderTest
   [junit4]   2> Creating dataDir: C:\Users\jenkins\workspace\Lucene-Solr-7.3-Windows\solr\build\solr-core\test\J0\temp\solr.core.ExitableDirectoryReaderTest_193A467CB4A58EEB-001\init-core-data-001
   [junit4]   2> 3129764 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 3129765 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason="", value=0.0/0.0, ssl=0.0/0.0, clientAuth=0.0/0.0)
   [junit4]   2> 3129765 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 3129766 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.SolrTestCaseJ4 ####initCore
   [junit4]   2> 3129766 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/C:/Users/jenkins/workspace/Lucene-Solr-7.3-Windows/solr/core/src/test-files/solr/collection1/lib, /C:/Users/jenkins/workspace/Lucene-Solr-7.3-Windows/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 3129780 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
   [junit4]   2> 3129785 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.s.IndexSchema [null] Schema name=example
   [junit4]   2> 3129802 ERROR (Finalizer) [    ] o.a.s.c.CoreContainer CoreContainer was not close prior to finalize(), indicates a bug -- POSSIBLE RESOURCE LEAK!!!  instance=1230113454
   [junit4]   2> 3129802 ERROR (Finalizer) [    ] o.a.s.c.CoreContainer CoreContainer was not close prior to finalize(), indicates a bug -- POSSIBLE RESOURCE LEAK!!!  instance=1413214296
   [junit4]   2> 3129802 ERROR (Finalizer) [    ] o.a.s.c.CoreContainer CoreContainer was not close prior to finalize(), indicates a bug -- POSSIBLE RESOURCE LEAK!!!  instance=1630450505
   [junit4]   2> 3129835 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.s.IndexSchema Loaded schema example/1.6 with uniqueid field id
   [junit4]   2> 3129879 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@671a8cf5
   [junit4]   2> 3129886 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@671a8cf5
   [junit4]   2> 3129886 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@671a8cf5
   [junit4]   2> 3129888 INFO  (coreLoadExecutor-10505-thread-1) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 3129889 INFO  (coreLoadExecutor-10505-thread-1) [    x:collection1] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/C:/Users/jenkins/workspace/Lucene-Solr-7.3-Windows/solr/core/src/test-files/solr/collection1/lib, /C:/Users/jenkins/workspace/Lucene-Solr-7.3-Windows/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 3129905 INFO  (coreLoadExecutor-10505-thread-1) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
   [junit4]   2> 3129911 INFO  (coreLoadExecutor-10505-thread-1) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=example
   [junit4]   2> 3129961 INFO  (coreLoadExecutor-10505-thread-1) [    x:collection1] o.a.s.s.IndexSchema Loaded schema example/1.6 with uniqueid field id
   [junit4]   2> 3129970 INFO  (coreLoadExecutor-10505-thread-1) [    x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir C:\Users\jenkins\workspace\Lucene-Solr-7.3-Windows\solr\core\src\test-files\solr\collection1, trusted=true
   [junit4]   2> 3129971 INFO  (coreLoadExecutor-10505-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@671a8cf5
   [junit4]   2> 3129971 INFO  (coreLoadExecutor-10505-thread-1) [    x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 3129971 INFO  (coreLoadExecutor-10505-thread-1) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [C:\Users\jenkins\workspace\Lucene-Solr-7.3-Windows\solr\core\src\test-files\solr\collection1], dataDir=[C:\Users\jenkins\workspace\Lucene-Solr-7.3-Windows\solr\build\solr-core\test\J0\temp\solr.core.ExitableDirectoryReaderTest_193A467CB4A58EEB-001\init-core-data-001\]
   [junit4]   2> 3129972 INFO  (coreLoadExecutor-10505-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=20, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 3130016 INFO  (coreLoadExecutor-10505-thread-1) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 3130016 INFO  (coreLoadExecutor-10505-thread-1) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 3130016 INFO  (coreLoadExecutor-10505-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=658629955, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 3130017 INFO  (coreLoadExecutor-10505-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@499c2db0[collection1] main]
   [junit4]   2> 3130017 WARN  (coreLoadExecutor-10505-thread-1) [    x:collection1] o.a.s.r.ManagedResourceStorage Cannot write to config directory C:\Users\jenkins\workspace\Lucene-Solr-7.3-Windows\solr\core\src\test-files\solr\collection1\conf; switching to use InMemory storage instead.
   [junit4]   2> 3130017 INFO  (coreLoadExecutor-10505-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 3130018 INFO  (searcherExecutor-10506-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@499c2db0[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 3130019 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.SolrTestCaseJ4 ####initCore end
   [junit4]   2> 3130020 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[0]} 0 0
   [junit4]   2> 3130021 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[1]} 0 0
   [junit4]   2> 3130021 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[2]} 0 0
   [junit4]   2> 3130021 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[3]} 0 0
   [junit4]   2> 3130022 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[4]} 0 0
   [junit4]   2> 3130022 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[5]} 0 0
   [junit4]   2> 3130022 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[6]} 0 0
   [junit4]   2> 3130022 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[7]} 0 0
   [junit4]   2> 3130022 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[8]} 0 0
   [junit4]   2> 3130022 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[9]} 0 0
   [junit4]   2> 3130022 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[10]} 0 0
   [junit4]   2> 3130023 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[11]} 0 0
   [junit4]   2> 3130023 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[12]} 0 0
   [junit4]   2> 3130023 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[13]} 0 0
   [junit4]   2> 3130023 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[14]} 0 0
   [junit4]   2> 3130023 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[15]} 0 0
   [junit4]   2> 3130023 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[16]} 0 0
   [junit4]   2> 3130024 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[17]} 0 0
   [junit4]   2> 3130024 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[18]} 0 0
   [junit4]   2> 3130024 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[19]} 0 0
   [junit4]   2> 3130024 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[20]} 0 0
   [junit4]   2> 3130024 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[21]} 0 0
   [junit4]   2> 3130024 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[22]} 0 0
   [junit4]   2> 3130024 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[23]} 0 0
   [junit4]   2> 3130025 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[24]} 0 0
   [junit4]   2> 3130025 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[25]} 0 0
   [junit4]   2> 3130025 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[26]} 0 0
   [junit4]   2> 3130025 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[27]} 0 0
   [junit4]   2> 3130025 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[28]} 0 0
   [junit4]   2> 3130025 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[29]} 0 0
   [junit4]   2> 3130025 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[30]} 0 0
   [junit4]   2> 3130025 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[31]} 0 0
   [junit4]   2> 3130025 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[32]} 0 0
   [junit4]   2> 3130025 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[33]} 0 0
   [junit4]   2> 3130025 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[34]} 0 0
   [junit4]   2> 3130025 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[35]} 0 0
   [junit4]   2> 3130025 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[36]} 0 0
   [junit4]   2> 3130025 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[37]} 0 0
   [junit4]   2> 3130025 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[38]} 0 0
   [junit4]   2> 3130025 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[39]} 0 0
   [junit4]   2> 3130025 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[40]} 0 0
   [junit4]   2> 3130025 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[41]} 0 0
   [junit4]   2> 3130025 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[42]} 0 0
   [junit4]   2> 3130025 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[43]} 0 0
   [junit4]   2> 3130025 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[44]} 0 0
   [junit4]   2> 3130025 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[45]} 0 0
   [junit4]   2> 3130025 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[46]} 0 0
   [junit4]   2> 3130025 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[47]} 0 0
   [junit4]   2> 3130025 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[48]} 0 0
   [junit4]   2> 3130025 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[49]} 0 0
   [junit4]   2> 3130025 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[50]} 0 0
   [junit4]   2> 3130025 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 3130029 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@17adf768 commitCommandVersion:0
   [junit4]   2> 3130033 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@609bfcf[collection1] main]
   [junit4]   2> 3130033 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 3130034 INFO  (searcherExecutor-10506-thread-1) [    ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@609bfcf[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.3.0):c51)))}
   [junit4]   2> 3130034 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{commit=} 0 5
   [junit4]   2> 3130035 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[51]} 0 0
   [junit4]   2> 3130035 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[52]} 0 0
   [junit4]   2> 3130035 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[53]} 0 0
   [junit4]   2> 3130035 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[54]} 0 0
   [junit4]   2> 3130036 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[55]} 0 0
   [junit4]   2> 3130036 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[56]} 0 0
   [junit4]   2> 3130036 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[57]} 0 0
   [junit4]   2> 3130036 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[58]} 0 0
   [junit4]   2> 3130036 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[59]} 0 0
   [junit4]   2> 3130037 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[60]} 0 0
   [junit4]   2> 3130037 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[61]} 0 0
   [junit4]   2> 3130037 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[62]} 0 0
   [junit4]   2> 3130037 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[63]} 0 0
   [junit4]   2> 3130038 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[64]} 0 0
   [junit4]   2> 3130038 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[65]} 0 0
   [junit4]   2> 3130038 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[66]} 0 0
   [junit4]   2> 3130038 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[67]} 0 0
   [junit4]   2> 3130038 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[68]} 0 0
   [junit4]   2> 3130038 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[69]} 0 0
   [junit4]   2> 3130038 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[70]} 0 0
   [junit4]   2> 3130038 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[71]} 0 0
   [junit4]   2> 3130038 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[72]} 0 0
   [junit4]   2> 3130038 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[73]} 0 0
   [junit4]   2> 3130039 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[74]} 0 0
   [junit4]   2> 3130039 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[75]} 0 0
   [junit4]   2> 3130039 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[76]} 0 0
   [junit4]   2> 3130039 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[77]} 0 0
   [junit4]   2> 3130039 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[78]} 0 0
   [junit4]   2> 3130039 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[79]} 0 0
   [junit4]   2> 3130039 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[80]} 0 0
   [junit4]   2> 3130040 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[81]} 0 0
   [junit4]   2> 3130040 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[82]} 0 0
   [junit4]   2> 3130040 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[83]} 0 0
   [junit4]   2> 3130040 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[84]} 0 0
   [junit4]   2> 3130040 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[85]} 0 0
   [junit4]   2> 3130040 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[86]} 0 0
   [junit4]   2> 3130040 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[87]} 0 0
   [junit4]   2> 3130041 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[88]} 0 0
   [junit4]   2> 3130041 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[89]} 0 0
   [junit4]   2> 3130041 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[90]} 0 0
   [junit4]   2> 3130041 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[91]} 0 0
   [junit4]   2> 3130041 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[92]} 0 0
   [junit4]   2> 3130041 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[93]} 0 0
   [junit4]   2> 3130041 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[94]} 0 0
   [junit4]   2> 3130042 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[95]} 0 0
   [junit4]   2> 3130042 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[96]} 0 0
   [junit4]   2> 3130042 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[97]} 0 0
   [junit4]   2> 3130042 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[98]} 0 0
   [junit4]   2> 3130042 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[99]} 0 0
   [junit4]   2> 3130042 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 3130042 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@17adf768 commitCommandVersion:0
   [junit4]   2> 3130044 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@3becb3a6[collection1] main]
   [junit4]   2> 3130045 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 3130045 INFO  (searcherExecutor-10506-thread-1) [    ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@3becb3a6[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.3.0):c51) Uninverting(_1(7.3.0):c49)))}
   [junit4]   2> 3130046 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{commit=} 0 3
   [junit4]   2> 3130048 INFO  (TEST-ExitableDirectoryReaderTest.testPrefixQuery-seed#[193A467CB4A58EEB]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testPrefixQuery
   [junit4]   2> 3130054 WARN  (TEST-ExitableDirectoryReaderTest.testPrefixQuery-seed#[193A467CB4A58EEB]) [    ] o.a.s.s.SolrIndexSearcher Query: [name:a*]; The request took too long to iterate over terms. Timeout: timeoutAt: 152831082696734 (System.nanoTime(): 152831087967598), TermsEnum=org.apache.lucene.codecs.memory.FSTOrdTermsReader$TermsReader$IntersectTermsEnum@717f8be5
   [junit4]   2> 3130055 INFO  (TEST-ExitableDirectoryReaderTest.testPrefixQuery-seed#[193A467CB4A58EEB]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={sleep=2&q=name:a*&timeAllowed=1&wt=xml} hits=0 status=0 QTime=7
   [junit4]   2> 3130056 INFO  (TEST-ExitableDirectoryReaderTest.testPrefixQuery-seed#[193A467CB4A58EEB]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=name:a*&timeAllowed=10000&wt=xml} hits=100 status=0 QTime=1
   [junit4]   2> 3130059 INFO  (TEST-ExitableDirectoryReaderTest.testPrefixQuery-seed#[193A467CB4A58EEB]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={sleep=2&q=name:a*&timeAllowed=1&wt=xml} hits=100 status=0 QTime=2
   [junit4]   2> 3130062 WARN  (TEST-ExitableDirectoryReaderTest.testPrefixQuery-seed#[193A467CB4A58EEB]) [    ] o.a.s.h.c.SearchHandler Query: sleep=2&q=*:*&fq=name:a*&timeAllowed=1&wt=xml; The request took too long to iterate over terms. Timeout: timeoutAt: 152831093596648 (System.nanoTime(): 152831094365893), TermsEnum=org.apache.lucene.codecs.memory.FSTOrdTermsReader$TermsReader$IntersectTermsEnum@7aff0247
   [junit4]   2> 3130062 INFO  (TEST-ExitableDirectoryReaderTest.testPrefixQuery-seed#[193A467CB4A58EEB]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={sleep=2&q=*:*&fq=name:a*&timeAllowed=1&wt=xml} status=0 QTime=2
   [junit4]   2> 3130062 INFO  (TEST-ExitableDirectoryReaderTest.testPrefixQuery-seed#[193A467CB4A58EEB]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=*:*&fq=name:a*&timeAllowed=10000&wt=xml} hits=100 status=0 QTime=0
   [junit4]   2> 3130062 INFO  (TEST-ExitableDirectoryReaderTest.testPrefixQuery-seed#[193A467CB4A58EEB]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=name:b*&timeAllowed=9223372036854775807&wt=xml} hits=100 status=0 QTime=0
   [junit4]   2> 3130062 INFO  (TEST-ExitableDirectoryReaderTest.testPrefixQuery-seed#[193A467CB4A58EEB]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=name:c*&timeAllowed=-7&wt=xml} hits=100 status=0 QTime=0
   [junit4]   2> 3130062 INFO  (TEST-ExitableDirectoryReaderTest.testPrefixQuery-seed#[193A467CB4A58EEB]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testPrefixQuery
   [junit4]   2> 3130062 INFO  (TEST-ExitableDirectoryReaderTest.testQueryResults-seed#[193A467CB4A58EEB]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testQueryResults
   [junit4]   2> 3130066 INFO  (TEST-ExitableDirectoryReaderTest.testQueryResults-seed#[193A467CB4A58EEB]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={sleep=2&q=name:e*&indent=true&timeAllowed=1&wt=xml} hits=100 status=0 QTime=0
   [junit4]   2> 3130066 INFO  (TEST-ExitableDirectoryReaderTest.testQueryResults-seed#[193A467CB4A58EEB]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testQueryResults
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ExitableDirectoryReaderTest -Dtests.method=testQueryResults -Dtests.seed=193A467CB4A58EEB -Dtests.slow=true -Dtests.locale=fr-ML -Dtests.timezone=Chile/Continental -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] FAILURE 0.01s J0 | ExitableDirectoryReaderTest.testQueryResults <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: Should NOT have inserted partial results! expected:<4> but was:<5>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([193A467CB4A58EEB:44B37E0516AED0A0]:0)
   [junit4]    > 	at org.apache.solr.core.ExitableDirectoryReaderTest.testQueryResults(ExitableDirectoryReaderTest.java:141)
   [junit4]    > 	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]    > 	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]    > 	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]    > 	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
   [junit4]    > 	at java.base/java.lang.Thread.run(Thread.java:844)
   [junit4]   2> 3130069 INFO  (TEST-ExitableDirectoryReaderTest.testCacheAssumptions-seed#[193A467CB4A58EEB]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testCacheAssumptions
   [junit4]   2> 3130071 WARN  (TEST-ExitableDirectoryReaderTest.testCacheAssumptions-seed#[193A467CB4A58EEB]) [    ] o.a.s.h.c.SearchHandler Query: sleep=2&q=*:*&indent=true&fq=name:d*&timeAllowed=1&wt=xml; The request took too long to iterate over terms. Timeout: timeoutAt: 152831103540829 (System.nanoTime(): 152831103957931), TermsEnum=org.apache.lucene.codecs.memory.FSTOrdTermsReader$TermsReader$IntersectTermsEnum@6361c6de
   [junit4]   2> 3130071 INFO  (TEST-ExitableDirectoryReaderTest.testCacheAssumptions-seed#[193A467CB4A58EEB]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={sleep=2&q=*:*&indent=true&fq=name:d*&timeAllowed=1&wt=xml} status=0 QTime=1
   [junit4]   2> 3130072 INFO  (TEST-ExitableDirectoryReaderTest.testCacheAssumptions-seed#[193A467CB4A58EEB]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=*:*&indent=true&fq=name:d*&timeAllowed=10000&wt=xml} hits=100 status=0 QTime=0
   [junit4]   2> 3130072 INFO  (TEST-ExitableDirectoryReaderTest.testCacheAssumptions-seed#[193A467CB4A58EEB]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testCacheAssumptions
   [junit4]   2> 3130072 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> 3130072 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=296519917
   [junit4]   2> 3130072 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 3130072 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@422e7f84: rootName = null, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@47a4fa5b
   [junit4]   2> 3130075 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 3130075 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@59a12467: rootName = null, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@101b32f5
   [junit4]   2> 3130078 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 3130078 INFO  (SUITE-ExitableDirectoryReaderTest-seed#[193A467CB4A58EEB]-worker) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@62605955: rootName = null, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@70aed8c5
   [junit4]   2> 3130078 INFO  (coreCloseExecutor-10510-thread-1) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@2806a99d
   [junit4]   2> 3130078 INFO  (coreCloseExecutor-10510-thread-1) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=671525277
   [junit4]   2> 3130078 INFO  (coreCloseExecutor-10510-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@629a9e06: rootName = null, domain = solr.core.collection1, service url = null, agent id = null] for registry solr.core.collection1 / com.codahale.metrics.MetricRegistry@51066606
   [junit4]   2> NOTE: leaving temporary files on disk at: C:\Users\jenkins\workspace\Lucene-Solr-7.3-Windows\solr\build\solr-core\test\J0\temp\solr.core.ExitableDirectoryReaderTest_193A467CB4A58EEB-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {name=FSTOrd50, id=FSTOrd50, text=PostingsFormat(name=Memory)}, docValues:{}, maxPointsInLeafNode=859, maxMBSortInHeap=7.271656288095519, sim=RandomSimilarity(queryNorm=true): {}, locale=fr-ML, timezone=Chile/Continental
   [junit4]   2> NOTE: Windows 10 10.0 amd64/Oracle Corporation 9.0.1 (64-bit)/cpus=3,threads=2,free=127747928,total=518979584
   [junit4]   2> NOTE: All tests run in this JVM: [SpatialFilterTest, SolrGangliaReporterTest, TestDeleteCollectionOnDownNodes, TestSortableTextField, UtilsToolTest, SolrPluginUtilsTest, OpenExchangeRatesOrgProviderTest, RemoteQueryErrorTest, TestSolrCloudWithHadoopAuthPlugin, TestDistributedMap, TestScoreJoinQPNoScore, TestInitQParser, HdfsTlogReplayBufferedWhileIndexingTest, CdcrBootstrapTest, CollectionPropsTest, TestExactSharedStatsCache, TriggerIntegrationTest, ResourceLoaderTest, XmlUpdateRequestHandlerTest, TimeRoutedAliasUpdateProcessorTest, TestSubQueryTransformerDistrib, TestFieldTypeCollectionResource, TestElisionMultitermQuery, SimpleMLTQParserTest, CollectionsAPISolrJTest, TestFieldCacheSortRandom, DocValuesMissingTest, AnalysisErrorHandlingTest, TestExclusionRuleCollectionAccess, TestDelegationWithHadoopAuth, TestFieldCacheWithThreads, BaseCdcrDistributedZkTest, TestOmitPositions, TestImplicitCoreProperties, TestIndexSearcher, TestSQLHandlerNonCloud, TemplateUpdateProcessorTest, DateMathParserTest, SolrJmxReporterCloudTest, TestMinMaxOnMultiValuedField, LeaderElectionIntegrationTest, HLLUtilTest, UninvertDocValuesMergePolicyTest, DistribJoinFromCollectionTest, AutoscalingHistoryHandlerTest, BasicZkTest, TestSolrCloudWithKerberosAlt, TestLRUStatsCache, TestLazyCores, DataDrivenBlockJoinTest, CopyFieldTest, TestTolerantSearch, SubstringBytesRefFilterTest, SuggesterTSTTest, DistributedMLTComponentTest, BlobRepositoryCloudTest, SolrRequestParserTest, TestReloadAndDeleteDocs, TestUnifiedSolrHighlighter, DeleteShardTest, TestManagedSynonymGraphFilterFactory, RecoveryZkTest, CollectionsAPIDistributedZkTest, PolyFieldTest, TestPerFieldSimilarityWithDefaultOverride, SystemLogListenerTest, TestComplexPhraseQParserPlugin, ChaosMonkeySafeLeaderWithPullReplicasTest, CloneFieldUpdateProcessorFactoryTest, QueryResultKeyTest, CoreAdminRequestStatusTest, SpatialHeatmapFacetsTest, TestStressLiveNodes, LIRRollingUpdatesTest, ActionThrottleTest, HdfsSyncSliceTest, TestPushWriter, TestInPlaceUpdatesDistrib, HdfsNNFailoverTest, AlternateDirectoryTest, HdfsRecoveryZkTest, CustomHighlightComponentTest, CheckHdfsIndexTest, DateFieldTest, TestSuggestSpellingConverter, DistributedFacetPivotWhiteBoxTest, TestFieldCacheReopen, TestPseudoReturnFields, TestSSLRandomization, SuggestComponentTest, TestRuleBasedAuthorizationPlugin, PropertiesRequestHandlerTest, CustomCollectionTest, LeaderVoteWaitTimeoutTest, ScheduledTriggerTest, LukeRequestHandlerTest, SpatialRPTFieldTypeTest, OpenCloseCoreStressTest, TestShortCircuitedRequests, TestTolerantUpdateProcessorCloud, TestUtilizeNode, JvmMetricsTest, TestEmbeddedSolrServerConstructors, ReplaceNodeTest, TestSubQueryTransformer, AsyncCallRequestStatusResponseTest, SuggesterFSTTest, DistributedQueryComponentOptimizationTest, TestRandomFlRTGCloud, AddSchemaFieldsUpdateProcessorFactoryTest, MoveReplicaTest, TestReplicaProperties, TestGraphMLResponseWriter, TestTrie, CursorMarkTest, TestXmlQParser, ZkCLITest, TestObjectReleaseTracker, SynonymTokenizerTest, AtomicUpdatesTest, CdcrVersionReplicationTest, PingRequestHandlerTest, TestSolr4Spatial2, SchemaVersionSpecificBehaviorTest, TestCoreBackup, TestLuceneIndexBackCompat, MergeStrategyTest, TestCloudJSONFacetJoinDomain, DistributedQueryElevationComponentTest, TestRawTransformer, TestInitParams, HttpSolrCallGetCoreTest, TestCloudPseudoReturnFields, TestPullReplica, AssignTest, TestAuthorizationFramework, TestRandomCollapseQParserPlugin, QueryElevationComponentTest, TestPivotHelperCode, TestFaceting, NumericFieldsTest, TestFieldCollectionResource, TestSimDistributedQueue, OverriddenZkACLAndCredentialsProvidersTest, TestCSVResponseWriter, TestLargeCluster, TestSolrCLIRunExample, TestUninvertingReader, TestSimpleQParserPlugin, BasicAuthStandaloneTest, ExecutePlanActionTest, TestConfigSetsAPIZkFailure, TestLegacyFieldReuse, SolrMetricReporterTest, TestLeaderElectionZkExpiry, SOLR749Test, TestPostingsSolrHighlighter, HighlighterTest, StatsComponentTest, HdfsChaosMonkeyNothingIsSafeTest, TestCoreContainer, CurrencyFieldTypeTest, ManagedSchemaRoundRobinCloudTest, TestPhraseSuggestions, FacetPivotSmallTest, TestFastOutputStream, SpellingQueryConverterTest, TestXIncludeConfig, ZkNodePropsTest, FileUtilsTest, OverseerModifyCollectionTest, TestExceedMaxTermLength, TestPerFieldSimilarity, TestConfigSetsAPI, TestPayloadScoreQParserPlugin, TestReRankQParserPlugin, TestFunctionQuery, ForceLeaderTest, TestOnReconnectListenerSupport, SolrIndexConfigTest, TestCloudManagedSchema, TestSimpleTextCodec, SolrIndexSplitterTest, SearchHandlerTest, FullHLLTest, TestSweetSpotSimilarityFactory, TermsComponentTest, TestLFUCache, SolrIndexMetricsTest, JSONWriterTest, TermVectorComponentDistributedTest, SolrSlf4jReporterTest, BasicDistributedZkTest, TestClusterProperties, TestConfigSetsAPIExclusivity, TestLeaderElectionWithEmptyReplica, TestLeaderInitiatedRecoveryThread, TestRebalanceLeaders, TestRequestForwarding, TestSegmentSorting, TestSizeLimitedDistributedMap, TestSkipOverseerOperations, TestSolrCloudWithDelegationTokens, TestSolrCloudWithSecureImpersonation, TestStressCloudBlindAtomicUpdates, TestTlogReplica, ZkFailoverTest, TestCollectionAPI, TestCollectionsAPIViaSolrCloudCluster, AutoAddReplicasIntegrationTest, AutoScalingHandlerTest, ComputePlanActionTest, SearchRateTriggerTest, TestComputePlanAction, TestNodeAddedTrigger, TestTriggerIntegration, CdcrReplicationHandlerTest, HDFSCollectionsAPITest, HdfsBasicDistributedZk2Test, HdfsRestartWhileUpdatingTest, HdfsWriteToMultipleCollectionsTest, StressHdfsTest, TestClusterStateMutator, ZkStateWriterTest, RuleEngineTest, CachingDirectoryFactoryTest, ConfigureRecoveryStrategyTest, CoreSorterTest, DirectoryFactoryTest, ExitableDirectoryReaderTest]
   [junit4] Completed [625/783 (1!)] on J0 in 0.33s, 3 tests, 1 failure <<< FAILURES!

[...truncated 43154 lines...]
[repro] Jenkins log URL: https://jenkins.thetaphi.de/job/Lucene-Solr-7.3-Windows/5/consoleText

[repro] Revision: 3b8eb6cd3e53727812f82711b9aa96d6f511e184

[repro] Ant options: "-Dargs=-XX:+UseCompressedOops -XX:+UseSerialGC"

[repro] ant clean

[...truncated 6 lines...]
[repro] Test suites by module:
[repro]    solr\core
[repro]       ExitableDirectoryReaderTest
[repro] ant compile-test

[...truncated 3336 lines...]
[repro] ant test-nocompile -Dtests.dups=5 -Dtests.maxfailures=5 -Dtests.class="*.ExitableDirectoryReaderTest" -Dtests.showOutput=onerror "-Dargs=-XX:+UseCompressedOops -XX:+UseSerialGC" -Dtests.seed=193A467CB4A58EEB -Dtests.slow=true -Dtests.locale=fr-ML -Dtests.timezone=Chile/Continental -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1

[...truncated 84 lines...]
[repro] Failures:
[repro]   0/5 failed: org.apache.solr.core.ExitableDirectoryReaderTest
[repro] Exiting with code 0

[...truncated 75 lines...]