You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2018/03/01 09:32:24 UTC
[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1491 - Still
Unstable
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1491/
4 tests failed.
FAILED: org.apache.solr.cloud.hdfs.HdfsTlogReplayBufferedWhileIndexingTest.test
Error Message:
Error from server at http://127.0.0.1:46131/hy_l: ADDREPLICA failed to create replica
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:46131/hy_l: ADDREPLICA failed to create replica
at __randomizedtesting.SeedInfo.seed([56FA8E5972B943D3:DEAEB183DC452E2B]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1104)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:884)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:817)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:425)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:341)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:991)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsTlogReplayBufferedWhileIndexingTest
Error Message:
37 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsTlogReplayBufferedWhileIndexingTest: 1) Thread[id=51272, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=51276, name=TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[56FA8E5972B943D3]-SendThread(127.0.0.1:42051), state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063) 3) Thread[id=51229, name=qtp719499696-51229, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=51223, name=qtp719499696-51223, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=51243, name=Thread-36255, state=WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1729) 6) Thread[id=51230, name=Scheduler-1151537675, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=51258, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 8) Thread[id=51226, name=qtp719499696-51226-acceptor-0@63439f95-ServerConnector@5517f1d8{HTTP/1.1,[http/1.1]}{127.0.0.1:54978}, state=RUNNABLE, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:379) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:638) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 9) Thread[id=51265, name=qtp1541700900-51265, state=RUNNABLE, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 10) Thread[id=51260, name=qtp1541700900-51260, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663) at java.lang.Thread.run(Thread.java:748) 11) Thread[id=51222, name=qtp719499696-51222, state=RUNNABLE, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 12) Thread[id=51266, name=qtp1541700900-51266, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 13) Thread[id=51245, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 14) Thread[id=51277, name=TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[56FA8E5972B943D3]-EventThread, state=WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502) 15) Thread[id=51256, name=searcherExecutor-7805-thread-1, state=WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 16) Thread[id=51259, name=qtp1541700900-51259, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663) at java.lang.Thread.run(Thread.java:748) 17) Thread[id=51234, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 18) Thread[id=51283, name=Scheduler-177000108, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 19) Thread[id=51224, name=qtp719499696-51224, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 20) Thread[id=51241, name=zkConnectionManagerCallback-4000-thread-1, state=WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 21) Thread[id=51280, name=Thread-36270, state=WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1729) 22) Thread[id=51225, name=qtp719499696-51225, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663) at java.lang.Thread.run(Thread.java:748) 23) Thread[id=51227, name=qtp719499696-51227, state=RUNNABLE, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140) at org.eclipse.jetty.io.ManagedSelector$$Lambda$22/1642071547.run(Unknown Source) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 24) Thread[id=51240, name=TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[56FA8E5972B943D3]-EventThread, state=WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502) 25) Thread[id=51278, name=zkConnectionManagerCallback-4010-thread-1, state=WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 26) Thread[id=51235, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 27) Thread[id=51261, name=qtp1541700900-51261-acceptor-0@387556bd-ServerConnector@290a2d6f{HTTP/1.1,[http/1.1]}{127.0.0.1:46131}, state=RUNNABLE, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:379) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:638) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 28) Thread[id=51262, name=qtp1541700900-51262, state=RUNNABLE, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 29) Thread[id=51228, name=qtp719499696-51228, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663) at java.lang.Thread.run(Thread.java:748) 30) Thread[id=51264, name=qtp1541700900-51264, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663) at java.lang.Thread.run(Thread.java:748) 31) Thread[id=51242, name=zkCallback-3999-thread-1, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 32) Thread[id=51263, name=qtp1541700900-51263, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663) at java.lang.Thread.run(Thread.java:748) 33) Thread[id=51282, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 34) Thread[id=51247, name=Scheduler-1310566548, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 35) Thread[id=51267, name=Scheduler-805509239, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 36) Thread[id=51239, name=TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[56FA8E5972B943D3]-SendThread(127.0.0.1:42051), state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1054) 37) Thread[id=51271, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 37 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsTlogReplayBufferedWhileIndexingTest:
1) Thread[id=51272, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=51276, name=TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[56FA8E5972B943D3]-SendThread(127.0.0.1:42051), state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)
at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
3) Thread[id=51229, name=qtp719499696-51229, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=51223, name=qtp719499696-51223, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=51243, name=Thread-36255, state=WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at org.apache.solr.core.CloserThread.run(CoreContainer.java:1729)
6) Thread[id=51230, name=Scheduler-1151537675, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=51258, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
8) Thread[id=51226, name=qtp719499696-51226-acceptor-0@63439f95-ServerConnector@5517f1d8{HTTP/1.1,[http/1.1]}{127.0.0.1:54978}, state=RUNNABLE, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:379)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:638)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
9) Thread[id=51265, name=qtp1541700900-51265, state=RUNNABLE, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
10) Thread[id=51260, name=qtp1541700900-51260, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
at java.lang.Thread.run(Thread.java:748)
11) Thread[id=51222, name=qtp719499696-51222, state=RUNNABLE, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
12) Thread[id=51266, name=qtp1541700900-51266, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
13) Thread[id=51245, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
14) Thread[id=51277, name=TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[56FA8E5972B943D3]-EventThread, state=WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)
15) Thread[id=51256, name=searcherExecutor-7805-thread-1, state=WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
16) Thread[id=51259, name=qtp1541700900-51259, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
at java.lang.Thread.run(Thread.java:748)
17) Thread[id=51234, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
18) Thread[id=51283, name=Scheduler-177000108, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
19) Thread[id=51224, name=qtp719499696-51224, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
20) Thread[id=51241, name=zkConnectionManagerCallback-4000-thread-1, state=WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
21) Thread[id=51280, name=Thread-36270, state=WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at org.apache.solr.core.CloserThread.run(CoreContainer.java:1729)
22) Thread[id=51225, name=qtp719499696-51225, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
at java.lang.Thread.run(Thread.java:748)
23) Thread[id=51227, name=qtp719499696-51227, state=RUNNABLE, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
at org.eclipse.jetty.io.ManagedSelector$$Lambda$22/1642071547.run(Unknown Source)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
24) Thread[id=51240, name=TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[56FA8E5972B943D3]-EventThread, state=WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)
25) Thread[id=51278, name=zkConnectionManagerCallback-4010-thread-1, state=WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
26) Thread[id=51235, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
27) Thread[id=51261, name=qtp1541700900-51261-acceptor-0@387556bd-ServerConnector@290a2d6f{HTTP/1.1,[http/1.1]}{127.0.0.1:46131}, state=RUNNABLE, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:379)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:638)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
28) Thread[id=51262, name=qtp1541700900-51262, state=RUNNABLE, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
29) Thread[id=51228, name=qtp719499696-51228, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
at java.lang.Thread.run(Thread.java:748)
30) Thread[id=51264, name=qtp1541700900-51264, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
at java.lang.Thread.run(Thread.java:748)
31) Thread[id=51242, name=zkCallback-3999-thread-1, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
32) Thread[id=51263, name=qtp1541700900-51263, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
at java.lang.Thread.run(Thread.java:748)
33) Thread[id=51282, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
34) Thread[id=51247, name=Scheduler-1310566548, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
35) Thread[id=51267, name=Scheduler-805509239, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
36) Thread[id=51239, name=TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[56FA8E5972B943D3]-SendThread(127.0.0.1:42051), state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1054)
37) Thread[id=51271, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([56FA8E5972B943D3]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsTlogReplayBufferedWhileIndexingTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=51222, name=qtp719499696-51222, state=RUNNABLE, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at sun.nio.ch.EPollArrayWrapper.interrupt(Native Method) at sun.nio.ch.EPollArrayWrapper.interrupt(EPollArrayWrapper.java:317) at sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:207) at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213) at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=51266, name=qtp1541700900-51266, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=51276, name=TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[56FA8E5972B943D3]-SendThread(127.0.0.1:42051), state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063) 4) Thread[id=51256, name=searcherExecutor-7805-thread-1, state=WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=51283, name=Scheduler-177000108, state=WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=51224, name=qtp719499696-51224, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=51241, name=zkConnectionManagerCallback-4000-thread-1, state=WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 8) Thread[id=51280, name=Thread-36270, state=WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1729) 9) Thread[id=51227, name=qtp719499696-51227, state=RUNNABLE, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at sun.nio.ch.IOUtil.drain(Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140) at org.eclipse.jetty.io.ManagedSelector$$Lambda$22/1642071547.run(Unknown Source) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 10) Thread[id=51278, name=zkConnectionManagerCallback-4010-thread-1, state=WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 11) Thread[id=51243, name=Thread-36255, state=WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1729) 12) Thread[id=51230, name=Scheduler-1151537675, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 13) Thread[id=51262, name=qtp1541700900-51262, state=RUNNABLE, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at sun.nio.ch.EPollArrayWrapper.interrupt(Native Method) at sun.nio.ch.EPollArrayWrapper.interrupt(EPollArrayWrapper.java:317) at sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:207) at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213) at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 14) Thread[id=51265, name=qtp1541700900-51265, state=RUNNABLE, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at sun.nio.ch.EPollArrayWrapper.interrupt(Native Method) at sun.nio.ch.EPollArrayWrapper.interrupt(EPollArrayWrapper.java:317) at sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:207) at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213) at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 15) Thread[id=51242, name=zkCallback-3999-thread-1, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 16) Thread[id=51247, name=Scheduler-1310566548, state=WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 17) Thread[id=51267, name=Scheduler-805509239, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 18) Thread[id=51239, name=TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[56FA8E5972B943D3]-SendThread(127.0.0.1:42051), state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=51222, name=qtp719499696-51222, state=RUNNABLE, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at sun.nio.ch.EPollArrayWrapper.interrupt(Native Method)
at sun.nio.ch.EPollArrayWrapper.interrupt(EPollArrayWrapper.java:317)
at sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:207)
at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=51266, name=qtp1541700900-51266, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=51276, name=TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[56FA8E5972B943D3]-SendThread(127.0.0.1:42051), state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)
at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
4) Thread[id=51256, name=searcherExecutor-7805-thread-1, state=WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=51283, name=Scheduler-177000108, state=WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=51224, name=qtp719499696-51224, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=51241, name=zkConnectionManagerCallback-4000-thread-1, state=WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
8) Thread[id=51280, name=Thread-36270, state=WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at org.apache.solr.core.CloserThread.run(CoreContainer.java:1729)
9) Thread[id=51227, name=qtp719499696-51227, state=RUNNABLE, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at sun.nio.ch.IOUtil.drain(Native Method)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
at org.eclipse.jetty.io.ManagedSelector$$Lambda$22/1642071547.run(Unknown Source)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
10) Thread[id=51278, name=zkConnectionManagerCallback-4010-thread-1, state=WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
11) Thread[id=51243, name=Thread-36255, state=WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at org.apache.solr.core.CloserThread.run(CoreContainer.java:1729)
12) Thread[id=51230, name=Scheduler-1151537675, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
13) Thread[id=51262, name=qtp1541700900-51262, state=RUNNABLE, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at sun.nio.ch.EPollArrayWrapper.interrupt(Native Method)
at sun.nio.ch.EPollArrayWrapper.interrupt(EPollArrayWrapper.java:317)
at sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:207)
at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
14) Thread[id=51265, name=qtp1541700900-51265, state=RUNNABLE, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at sun.nio.ch.EPollArrayWrapper.interrupt(Native Method)
at sun.nio.ch.EPollArrayWrapper.interrupt(EPollArrayWrapper.java:317)
at sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:207)
at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
15) Thread[id=51242, name=zkCallback-3999-thread-1, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
16) Thread[id=51247, name=Scheduler-1310566548, state=WAITING, group=TGRP-HdfsT
[...truncated too long message...]
> at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 11) Thread[id=51243, name=Thread-36255, state=WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
[junit4] > at java.lang.Object.wait(Native Method)
[junit4] > at java.lang.Object.wait(Object.java:502)
[junit4] > at org.apache.solr.core.CloserThread.run(CoreContainer.java:1729)
[junit4] > 12) Thread[id=51230, name=Scheduler-1151537675, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 13) Thread[id=51262, name=qtp1541700900-51262, state=RUNNABLE, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
[junit4] > at sun.nio.ch.EPollArrayWrapper.interrupt(Native Method)
[junit4] > at sun.nio.ch.EPollArrayWrapper.interrupt(EPollArrayWrapper.java:317)
[junit4] > at sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:207)
[junit4] > at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
[junit4] > at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
[junit4] > at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)
[junit4] > at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)
[junit4] > at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
[junit4] > at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)
[junit4] > at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 14) Thread[id=51265, name=qtp1541700900-51265, state=RUNNABLE, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
[junit4] > at sun.nio.ch.EPollArrayWrapper.interrupt(Native Method)
[junit4] > at sun.nio.ch.EPollArrayWrapper.interrupt(EPollArrayWrapper.java:317)
[junit4] > at sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:207)
[junit4] > at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
[junit4] > at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
[junit4] > at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)
[junit4] > at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)
[junit4] > at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
[junit4] > at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)
[junit4] > at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 15) Thread[id=51242, name=zkCallback-3999-thread-1, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 16) Thread[id=51247, name=Scheduler-1310566548, state=WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 17) Thread[id=51267, name=Scheduler-805509239, state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 18) Thread[id=51239, name=TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[56FA8E5972B943D3]-SendThread(127.0.0.1:42051), state=TIMED_WAITING, group=TGRP-HdfsTlogReplayBufferedWhileIndexingTest]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
[junit4] > at __randomizedtesting.SeedInfo.seed([56FA8E5972B943D3]:0)
[junit4] Completed [608/778 (2!)] on J2 in 93.08s, 1 test, 3 errors <<< FAILURES!
[...truncated 48320 lines...]
[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1504 - Failure
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1504/
5 tests failed.
FAILED: org.apache.lucene.search.TestInetAddressRangeQueries.testRandomBig
Error Message:
Java heap space
Stack Trace:
java.lang.OutOfMemoryError: Java heap space
at __randomizedtesting.SeedInfo.seed([E61A8AA73526F1C8:614DF728A47F8D48]:0)
at org.apache.lucene.util.FixedLengthBytesRefArray.append(FixedLengthBytesRefArray.java:83)
at org.apache.lucene.util.OfflineSorter.readPartition(OfflineSorter.java:424)
at org.apache.lucene.util.OfflineSorter.sort(OfflineSorter.java:274)
at org.apache.lucene.util.bkd.BKDWriter.sort(BKDWriter.java:914)
at org.apache.lucene.util.bkd.BKDWriter.finish(BKDWriter.java:990)
at org.apache.lucene.index.RandomCodec$1$1.writeField(RandomCodec.java:141)
at org.apache.lucene.codecs.PointsWriter.mergeOneField(PointsWriter.java:62)
at org.apache.lucene.codecs.PointsWriter.merge(PointsWriter.java:186)
at org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.merge(Lucene60PointsWriter.java:144)
at org.apache.lucene.codecs.asserting.AssertingPointsFormat$AssertingPointsWriter.merge(AssertingPointsFormat.java:142)
at org.apache.lucene.index.SegmentMerger.mergePoints(SegmentMerger.java:200)
at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:160)
at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4515)
at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4154)
at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2331)
at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:2164)
at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:2115)
at org.apache.lucene.search.BaseRangeFieldQueryTestCase.verify(BaseRangeFieldQueryTestCase.java:202)
at org.apache.lucene.search.BaseRangeFieldQueryTestCase.doTestRandom(BaseRangeFieldQueryTestCase.java:160)
at org.apache.lucene.search.BaseRangeFieldQueryTestCase.testRandomBig(BaseRangeFieldQueryTestCase.java:75)
at org.apache.lucene.search.TestInetAddressRangeQueries.testRandomBig(TestInetAddressRangeQueries.java:81)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
FAILED: org.apache.solr.cloud.CreateRoutedAliasTest.testNegativeFutureFails
Error Message:
Error from server at http://127.0.0.1:39211/solr: Collection : testV1_2018-03-17_17 is part of alias testV1 remove or modify the alias before removing this collection.
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:39211/solr: Collection : testV1_2018-03-17_17 is part of alias testV1 remove or modify the alias before removing this collection.
at __randomizedtesting.SeedInfo.seed([EB534B6BF3D0DA43:4785D3710189462A]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1105)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:885)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:818)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
at org.apache.solr.cloud.MiniSolrCloudCluster.deleteAllCollections(MiniSolrCloudCluster.java:451)
at org.apache.solr.cloud.CreateRoutedAliasTest.doBefore(CreateRoutedAliasTest.java:96)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:968)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.LIROnShardRestartTest.testAllReplicasInLIR
Error Message:
No live SolrServers available to handle this request
Stack Trace:
org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request
at __randomizedtesting.SeedInfo.seed([EB534B6BF3D0DA43:B1CB71AD8D50BDA4]:0)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:460)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1105)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:885)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:818)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
at org.apache.solr.cloud.LIROnShardRestartTest.testAllReplicasInLIR(LIROnShardRestartTest.java:160)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.update.TestInPlaceUpdatesDistrib.test
Error Message:
Error from server at http://127.0.0.1:40408/collection1: ERROR adding document SolrInputDocument(fields: [id=-9982, title_s=title-9982, id_i=-9982, _version_=1595208978055299072])
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:40408/collection1: ERROR adding document SolrInputDocument(fields: [id=-9982, title_s=title-9982, id_i=-9982, _version_=1595208978055299072])
at __randomizedtesting.SeedInfo.seed([EB534B6BF3D0DA43:630774B15D2CB7BB]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
at org.apache.solr.update.TestInPlaceUpdatesDistrib.addDocAndGetVersion(TestInPlaceUpdatesDistrib.java:1105)
at org.apache.solr.update.TestInPlaceUpdatesDistrib.buildRandomIndex(TestInPlaceUpdatesDistrib.java:1150)
at org.apache.solr.update.TestInPlaceUpdatesDistrib.docValuesUpdateTest(TestInPlaceUpdatesDistrib.java:318)
at org.apache.solr.update.TestInPlaceUpdatesDistrib.test(TestInPlaceUpdatesDistrib.java:144)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.AliasIntegrationTest.test
Error Message:
Error from server at https://127.0.0.1:37281/solr: Could not fully remove collection: collection1
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:37281/solr: Could not fully remove collection: collection1
at __randomizedtesting.SeedInfo.seed([EB534B6BF3D0DA43:630774B15D2CB7BB]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1105)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:885)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:818)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
at org.apache.solr.cloud.MiniSolrCloudCluster.deleteAllCollections(MiniSolrCloudCluster.java:451)
at org.apache.solr.cloud.AliasIntegrationTest.tearDown(AliasIntegrationTest.java:92)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 2150 lines...]
[junit4] JVM J1: stdout was not empty, see: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp/junit4-J1-20180317_091408_2078394789432174952604.sysout
[junit4] >>> JVM J1 emitted unexpected output (verbatim) ----
[junit4] codec: DummyCompressingStoredFields, pf: FSTOrd50, dvf: Asserting
[junit4] <<< JVM J1: EOF ----
[...truncated 7197 lines...]
[junit4] Suite: org.apache.lucene.search.TestInetAddressRangeQueries
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestInetAddressRangeQueries -Dtests.method=testRandomBig -Dtests.seed=E61A8AA73526F1C8 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=tr-TR -Dtests.timezone=Asia/Amman -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] ERROR 273s J1 | TestInetAddressRangeQueries.testRandomBig <<<
[junit4] > Throwable #1: java.lang.OutOfMemoryError: Java heap space
[junit4] > at __randomizedtesting.SeedInfo.seed([E61A8AA73526F1C8:614DF728A47F8D48]:0)
[junit4] > at org.apache.lucene.util.FixedLengthBytesRefArray.append(FixedLengthBytesRefArray.java:83)
[junit4] > at org.apache.lucene.util.OfflineSorter.readPartition(OfflineSorter.java:424)
[junit4] > at org.apache.lucene.util.OfflineSorter.sort(OfflineSorter.java:274)
[junit4] > at org.apache.lucene.util.bkd.BKDWriter.sort(BKDWriter.java:914)
[junit4] > at org.apache.lucene.util.bkd.BKDWriter.finish(BKDWriter.java:990)
[junit4] > at org.apache.lucene.index.RandomCodec$1$1.writeField(RandomCodec.java:141)
[junit4] > at org.apache.lucene.codecs.PointsWriter.mergeOneField(PointsWriter.java:62)
[junit4] > at org.apache.lucene.codecs.PointsWriter.merge(PointsWriter.java:186)
[junit4] > at org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.merge(Lucene60PointsWriter.java:144)
[junit4] > at org.apache.lucene.codecs.asserting.AssertingPointsFormat$AssertingPointsWriter.merge(AssertingPointsFormat.java:142)
[junit4] > at org.apache.lucene.index.SegmentMerger.mergePoints(SegmentMerger.java:200)
[junit4] > at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:160)
[junit4] > at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4515)
[junit4] > at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4154)
[junit4] > at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
[junit4] > at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2331)
[junit4] > at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:2164)
[junit4] > at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:2115)
[junit4] > at org.apache.lucene.search.BaseRangeFieldQueryTestCase.verify(BaseRangeFieldQueryTestCase.java:202)
[junit4] > at org.apache.lucene.search.BaseRangeFieldQueryTestCase.doTestRandom(BaseRangeFieldQueryTestCase.java:160)
[junit4] > at org.apache.lucene.search.BaseRangeFieldQueryTestCase.testRandomBig(BaseRangeFieldQueryTestCase.java:75)
[junit4] > at org.apache.lucene.search.TestInetAddressRangeQueries.testRandomBig(TestInetAddressRangeQueries.java:81)
[junit4] 2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/misc/test/J1/temp/lucene.search.TestInetAddressRangeQueries_E61A8AA73526F1C8-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {id=Lucene50(blocksize=128)}, docValues:{ipRangeField=DocValuesFormat(name=Direct), id=DocValuesFormat(name=Lucene70)}, maxPointsInLeafNode=1985, maxMBSortInHeap=7.490015597112068, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@6ab5e8a6), locale=tr-TR, timezone=Asia/Amman
[junit4] 2> NOTE: Linux 4.4.0-112-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=1,free=60551168,total=477626368
[junit4] 2> NOTE: All tests run in this JVM: [TestHighFreqTerms, TestInetAddressRangeQueries]
[junit4] Completed [13/13 (1!)] on J1 in 343.74s, 4 tests, 1 error <<< FAILURES!
[...truncated 1 lines...]
[junit4] JVM J1: stdout was not empty, see: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/misc/test/temp/junit4-J1-20180317_144157_0106798037024808875301.sysout
[junit4] >>> JVM J1 emitted unexpected output (verbatim) ----
[junit4] java.lang.OutOfMemoryError: Java heap space
[junit4] Dumping heap to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps/java_pid13381.hprof ...
[junit4] Heap dump file created [580222405 bytes in 3.591 secs]
[junit4] <<< JVM J1: EOF ----
[...truncated 5503 lines...]
[junit4] Suite: org.apache.solr.cloud.AliasIntegrationTest
[junit4] 2> 5330427 INFO (SUITE-AliasIntegrationTest-seed#[EB534B6BF3D0DA43]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.AliasIntegrationTest_EB534B6BF3D0DA43-001/init-core-data-001
[junit4] 2> 5330427 WARN (SUITE-AliasIntegrationTest-seed#[EB534B6BF3D0DA43]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=20 numCloses=20
[junit4] 2> 5330427 INFO (SUITE-AliasIntegrationTest-seed#[EB534B6BF3D0DA43]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 5330445 INFO (SUITE-AliasIntegrationTest-seed#[EB534B6BF3D0DA43]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 5330446 INFO (SUITE-AliasIntegrationTest-seed#[EB534B6BF3D0DA43]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.AliasIntegrationTest_EB534B6BF3D0DA43-001/tempDir-001
[junit4] 2> 5330446 INFO (SUITE-AliasIntegrationTest-seed#[EB534B6BF3D0DA43]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 5330447 INFO (Thread-11759) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 5330447 INFO (Thread-11759) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 5330461 ERROR (Thread-11759) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 5330566 INFO (SUITE-AliasIntegrationTest-seed#[EB534B6BF3D0DA43]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:36561
[junit4] 2> 5330574 INFO (zkConnectionManagerCallback-6071-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 5330579 INFO (jetty-launcher-6068-thread-1) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T23:27:37+02:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 5330587 INFO (jetty-launcher-6068-thread-2) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T23:27:37+02:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 5330652 INFO (jetty-launcher-6068-thread-1) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 5330652 INFO (jetty-launcher-6068-thread-1) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 5330652 INFO (jetty-launcher-6068-thread-1) [ ] o.e.j.s.session Scavenging every 600000ms
[junit4] 2> 5330652 INFO (jetty-launcher-6068-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@8331acd{/solr,null,AVAILABLE}
[junit4] 2> 5330653 INFO (jetty-launcher-6068-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@32b6429a{SSL,[ssl, http/1.1]}{127.0.0.1:37281}
[junit4] 2> 5330653 INFO (jetty-launcher-6068-thread-1) [ ] o.e.j.s.Server Started @5342090ms
[junit4] 2> 5330653 INFO (jetty-launcher-6068-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=37281}
[junit4] 2> 5330654 ERROR (jetty-launcher-6068-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 5330654 INFO (jetty-launcher-6068-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 8.0.0
[junit4] 2> 5330654 INFO (jetty-launcher-6068-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 5330654 INFO (jetty-launcher-6068-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 5330654 INFO (jetty-launcher-6068-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-03-17T17:12:16.781Z
[junit4] 2> 5330693 INFO (jetty-launcher-6068-thread-2) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 5330693 INFO (jetty-launcher-6068-thread-2) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 5330693 INFO (jetty-launcher-6068-thread-2) [ ] o.e.j.s.session Scavenging every 660000ms
[junit4] 2> 5330708 INFO (zkConnectionManagerCallback-6073-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 5330709 INFO (jetty-launcher-6068-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 5330722 INFO (jetty-launcher-6068-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2e4105fa{/solr,null,AVAILABLE}
[junit4] 2> 5330723 INFO (jetty-launcher-6068-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@d202f71{SSL,[ssl, http/1.1]}{127.0.0.1:45225}
[junit4] 2> 5330723 INFO (jetty-launcher-6068-thread-2) [ ] o.e.j.s.Server Started @5342160ms
[junit4] 2> 5330723 INFO (jetty-launcher-6068-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=45225}
[junit4] 2> 5330723 ERROR (jetty-launcher-6068-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 5330723 INFO (jetty-launcher-6068-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 8.0.0
[junit4] 2> 5330723 INFO (jetty-launcher-6068-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 5330723 INFO (jetty-launcher-6068-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 5330724 INFO (jetty-launcher-6068-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-03-17T17:12:16.850Z
[junit4] 2> 5330737 INFO (zkConnectionManagerCallback-6075-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 5330747 INFO (jetty-launcher-6068-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 5330750 INFO (jetty-launcher-6068-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36561/solr
[junit4] 2> 5330773 INFO (zkConnectionManagerCallback-6079-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 5330776 INFO (zkConnectionManagerCallback-6082-thread-1-processing-n:127.0.0.1:37281_solr) [n:127.0.0.1:37281_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 5330839 INFO (jetty-launcher-6068-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36561/solr
[junit4] 2> 5330867 INFO (zkConnectionManagerCallback-6086-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 5330941 INFO (zkConnectionManagerCallback-6088-thread-1-processing-n:127.0.0.1:45225_solr) [n:127.0.0.1:45225_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 5330971 INFO (jetty-launcher-6068-thread-2) [n:127.0.0.1:45225_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 5330971 INFO (jetty-launcher-6068-thread-2) [n:127.0.0.1:45225_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:45225_solr
[junit4] 2> 5330972 INFO (jetty-launcher-6068-thread-2) [n:127.0.0.1:45225_solr ] o.a.s.c.Overseer Overseer (id=72296416233914374-127.0.0.1:45225_solr-n_0000000000) starting
[junit4] 2> 5331073 INFO (zkConnectionManagerCallback-6093-thread-1-processing-n:127.0.0.1:45225_solr) [n:127.0.0.1:45225_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 5331087 INFO (jetty-launcher-6068-thread-2) [n:127.0.0.1:45225_solr ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:36561/solr ready
[junit4] 2> 5331088 INFO (jetty-launcher-6068-thread-2) [n:127.0.0.1:45225_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45225_solr
[junit4] 2> 5331125 INFO (zkCallback-6092-thread-1-processing-n:127.0.0.1:45225_solr) [n:127.0.0.1:45225_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 5331160 INFO (zkCallback-6087-thread-1-processing-n:127.0.0.1:45225_solr) [n:127.0.0.1:45225_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 5331644 INFO (jetty-launcher-6068-thread-1) [n:127.0.0.1:37281_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 5331682 INFO (jetty-launcher-6068-thread-1) [n:127.0.0.1:37281_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 5331717 INFO (jetty-launcher-6068-thread-1) [n:127.0.0.1:37281_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37281_solr
[junit4] 2> 5331718 INFO (zkCallback-6092-thread-1-processing-n:127.0.0.1:45225_solr) [n:127.0.0.1:45225_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 5331718 INFO (zkCallback-6087-thread-1-processing-n:127.0.0.1:45225_solr) [n:127.0.0.1:45225_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 5331768 INFO (zkCallback-6081-thread-1-processing-n:127.0.0.1:37281_solr) [n:127.0.0.1:37281_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 5333037 INFO (jetty-launcher-6068-thread-1) [n:127.0.0.1:37281_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_37281.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@49922e4f
[junit4] 2> 5333106 INFO (jetty-launcher-6068-thread-1) [n:127.0.0.1:37281_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_37281.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@49922e4f
[junit4] 2> 5333106 INFO (jetty-launcher-6068-thread-1) [n:127.0.0.1:37281_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_37281.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@49922e4f
[junit4] 2> 5333107 INFO (jetty-launcher-6068-thread-1) [n:127.0.0.1:37281_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.AliasIntegrationTest_EB534B6BF3D0DA43-001/tempDir-001/node1/.
[junit4] 2> 5333246 INFO (zkConnectionManagerCallback-6097-thread-1-processing-n:127.0.0.1:37281_solr) [n:127.0.0.1:37281_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 5333267 INFO (jetty-launcher-6068-thread-1) [n:127.0.0.1:37281_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 5333268 INFO (jetty-launcher-6068-thread-1) [n:127.0.0.1:37281_solr ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:36561/solr ready
[junit4] 2> 5333717 INFO (jetty-launcher-6068-thread-2) [n:127.0.0.1:45225_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45225.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@49922e4f
[junit4] 2> 5333741 INFO (jetty-launcher-6068-thread-2) [n:127.0.0.1:45225_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45225.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@49922e4f
[junit4] 2> 5333741 INFO (jetty-launcher-6068-thread-2) [n:127.0.0.1:45225_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45225.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@49922e4f
[junit4] 2> 5333755 INFO (jetty-launcher-6068-thread-2) [n:127.0.0.1:45225_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.AliasIntegrationTest_EB534B6BF3D0DA43-001/tempDir-001/node2/.
[junit4] 2> 5333897 INFO (zkConnectionManagerCallback-6099-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 5334002 INFO (zkConnectionManagerCallback-6103-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 5334003 INFO (SUITE-AliasIntegrationTest-seed#[EB534B6BF3D0DA43]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 5334034 INFO (SUITE-AliasIntegrationTest-seed#[EB534B6BF3D0DA43]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:36561/solr ready
[junit4] 2> 5334179 INFO (TEST-AliasIntegrationTest.testErrorChecks-seed#[EB534B6BF3D0DA43]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testErrorChecks
[junit4] 2> 5334411 INFO (qtp143682733-28597) [n:127.0.0.1:37281_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf&name=testErrorChecks-collection&nrtReplicas=1&action=CREATE&numShards=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 5334445 INFO (OverseerThreadFactory-8022-thread-1-processing-n:127.0.0.1:45225_solr) [n:127.0.0.1:45225_solr ] o.a.s.c.a.c.CreateCollectionCmd Create collection testErrorChecks-collection
[junit4] 2> 5334624 INFO (OverseerStateUpdate-72296416233914374-127.0.0.1:45225_solr-n_0000000000) [n:127.0.0.1:45225_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"testErrorChecks-collection",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"testErrorChecks-collection_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:45225/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 5334626 INFO (OverseerStateUpdate-72296416233914374-127.0.0.1:45225_solr-n_0000000000) [n:127.0.0.1:45225_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"testErrorChecks-collection",
[junit4] 2> "shard":"shard2",
[junit4] 2> "core":"testErrorChecks-collection_shard2_replica_n2",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:37281/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 5335008 INFO (qtp143682733-28604) [n:127.0.0.1:37281_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=testErrorChecks-collection_shard2_replica_n2&action=CREATE&numShards=2&collection=testErrorChecks-collection&shard=shard2&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 5335017 INFO (qtp143682733-28604) [n:127.0.0.1:37281_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 5335085 INFO (qtp1607421543-28598) [n:127.0.0.1:45225_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=testErrorChecks-collection_shard1_replica_n1&action=CREATE&numShards=2&collection=testErrorChecks-collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 5335086 INFO (qtp1607421543-28598) [n:127.0.0.1:45225_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 5335244 INFO (zkCallback-6081-thread-1-processing-n:127.0.0.1:37281_solr) [n:127.0.0.1:37281_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testErrorChecks-collection/state.json] for collection [testErrorChecks-collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 5335245 INFO (zkCallback-6087-thread-1-processing-n:127.0.0.1:45225_solr) [n:127.0.0.1:45225_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testErrorChecks-collection/state.json] for collection [testErrorChecks-collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 5335248 INFO (zkCallback-6081-thread-2-processing-n:127.0.0.1:37281_solr) [n:127.0.0.1:37281_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testErrorChecks-collection/state.json] for collection [testErrorChecks-collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 5335264 INFO (zkCallback-6087-thread-2-processing-n:127.0.0.1:45225_solr) [n:127.0.0.1:45225_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testErrorChecks-collection/state.json] for collection [testErrorChecks-collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 5336165 INFO (qtp1607421543-28598) [n:127.0.0.1:45225_solr c:testErrorChecks-collection s:shard1 r:core_node3 x:testErrorChecks-collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 5336200 INFO (qtp143682733-28604) [n:127.0.0.1:37281_solr c:testErrorChecks-collection s:shard2 r:core_node4 x:testErrorChecks-collection_shard2_replica_n2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 5336358 INFO (qtp1607421543-28598) [n:127.0.0.1:45225_solr c:testErrorChecks-collection s:shard1 r:core_node3 x:testErrorChecks-collection_shard1_replica_n1] o.a.s.s.IndexSchema [testErrorChecks-collection_shard1_replica_n1] Schema name=minimal
[junit4] 2> 5336373 INFO (qtp143682733-28604) [n:127.0.0.1:37281_solr c:testErrorChecks-collection s:shard2 r:core_node4 x:testErrorChecks-collection_shard2_replica_n2] o.a.s.s.IndexSchema [testErrorChecks-collection_shard2_replica_n2] Schema name=minimal
[junit4] 2> 5336375 INFO (qtp143682733-28604) [n:127.0.0.1:37281_solr c:testErrorChecks-collection s:shard2 r:core_node4 x:testErrorChecks-collection_shard2_replica_n2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 5336375 INFO (qtp143682733-28604) [n:127.0.0.1:37281_solr c:testErrorChecks-collection s:shard2 r:core_node4 x:testErrorChecks-collection_shard2_replica_n2] o.a.s.c.CoreContainer Creating SolrCore 'testErrorChecks-collection_shard2_replica_n2' using configuration from collection testErrorChecks-collection, trusted=true
[junit4] 2> 5336376 INFO (qtp143682733-28604) [n:127.0.0.1:37281_solr c:testErrorChecks-collection s:shard2 r:core_node4 x:testErrorChecks-collection_shard2_replica_n2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_37281.solr.core.testErrorChecks-collection.shard2.replica_n2' (registry 'solr.core.testErrorChecks-collection.shard2.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@49922e4f
[junit4] 2> 5336393 INFO (qtp1607421543-28598) [n:127.0.0.1:45225_solr c:testErrorChecks-collection s:shard1 r:core_node3 x:testErrorChecks-collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 5336393 INFO (qtp1607421543-28598) [n:127.0.0.1:45225_solr c:testErrorChecks-collection s:shard1 r:core_node3 x:testErrorChecks-collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'testErrorChecks-collection_shard1_replica_n1' using configuration from collection testErrorChecks-collection, trusted=true
[junit4] 2> 5336393 INFO (qtp1607421543-28598) [n:127.0.0.1:45225_solr c:testErrorChecks-collection s:shard1 r:core_node3 x:testErrorChecks-collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45225.solr.core.testErrorChecks-collection.shard1.replica_n1' (registry 'solr.core.testErrorChecks-collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@49922e4f
[junit4] 2> 5336394 INFO (qtp1607421543-28598) [n:127.0.0.1:45225_solr c:testErrorChecks-collection s:shard1 r:core_node3 x:testErrorChecks-collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 5336394 INFO (qtp1607421543-28598) [n:127.0.0.1:45225_solr c:testErrorChecks-collection s:shard1 r:core_node3 x:testErrorChecks-collection_shard1_replica_n1] o.a.s.c.SolrCore [[testErrorChecks-collection_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.AliasIntegrationTest_EB534B6BF3D0DA43-001/tempDir-001/node2/testErrorChecks-collection_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.AliasIntegrationTest_EB534B6BF3D0DA43-001/tempDir-001/node2/./testErrorChecks-collection_shard1_replica_n1/data/]
[junit4] 2> 5336409 INFO (qtp143682733-28604) [n:127.0.0.1:37281_solr c:testErrorChecks-collection s:shard2 r:core_node4 x:testErrorChecks-collection_shard2_replica_n2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 5336409 INFO (qtp143682733-28604) [n:127.0.0.1:37281_solr c:testErrorChecks-collection s:shard2 r:core_node4 x:testErrorChecks-collection_shard2_replica_n2] o.a.s.c.SolrCore [[testErrorChecks-collection_shard2_replica_n2] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.AliasIntegrationTest_EB534B6BF3D0DA43-001/tempDir-001/node1/testErrorChecks-collection_shard2_replica_n2], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.AliasIntegrationTest_EB534B6BF3D0DA43-001/tempDir-001/node1/./testErrorChecks-collection_shard2_replica_n2/data/]
[junit4] 2> 5336761 INFO (qtp143682733-28604) [n:127.0.0.1:37281_solr c:testErrorChecks-collection s:shard2 r:core_node4 x:testErrorChecks-collection_shard2_replica_n2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 5336790 INFO (qtp143682733-28604) [n:127.0.0.1:37281_solr c:testErrorChecks-collection s:shard2 r:core_node4 x:testErrorChecks-collection_shard2_replica_n2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 5336791 INFO (qtp143682733-28604) [n:127.0.0.1:37281_solr c:testErrorChecks-collection s:shard2 r:core_node4 x:testErrorChecks-collection_shard2_replica_n2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 5336791 INFO (qtp143682733-28604) [n:127.0.0.1:37281_solr c:testErrorChecks-collection s:shard2 r:core_node4 x:testErrorChecks-collection_shard2_replica_n2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 5336793 INFO (qtp143682733-28604) [n:127.0.0.1:37281_solr c:testErrorChecks-collection s:shard2 r:core_node4 x:testErrorChecks-collection_shard2_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@74bb6756[testErrorChecks-collection_shard2_replica_n2] main]
[junit4] 2> 5336799 INFO (qtp143682733-28604) [n:127.0.0.1:37281_solr c:testErrorChecks-collection s:shard2 r:core_node4 x:testErrorChecks-collection_shard2_replica_n2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 5336799 INFO (qtp143682733-28604) [n:127.0.0.1:37281_solr c:testErrorChecks-collection s:shard2 r:core_node4 x:testErrorChecks-collection_shard2_replica_n2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 5336800 INFO (qtp143682733-28604) [n:127.0.0.1:37281_solr c:testErrorChecks-collection s:shard2 r:core_node4 x:testErrorChecks-collection_shard2_replica_n2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 5336800 INFO (qtp143682733-28604) [n:127.0.0.1:37281_solr c:testErrorChecks-collection s:shard2 r:core_node4 x:testErrorChecks-collection_shard2_replica_n2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1595205739271421952
[junit4] 2> 5336811 INFO (searcherExecutor-8027-thread-1-processing-n:127.0.0.1:37281_solr x:testErrorChecks-collection_shard2_replica_n2 s:shard2 c:testErrorChecks-collection r:core_node4) [n:127.0.0.1:37281_solr c:testErrorChecks-collection s:shard2 r:core_node4 x:testErrorChecks-collection_shard2_replica_n2] o.a.s.c.SolrCore [testErrorChecks-collection_shard2_replica_n2] Registered new searcher Searcher@74bb6756[testErrorChecks-collection_shard2_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 5336833 INFO (qtp143682733-28604) [n:127.0.0.1:37281_solr c:testErrorChecks-collection s:shard2 r:core_node4 x:testErrorChecks-collection_shard2_replica_n2] o.a.s.c.ZkShardTerms Successful update of terms at /collections/testErrorChecks-collection/terms/shard2 to Terms{values={core_node4=0}, version=0}
[junit4] 2> 5336835 INFO (qtp143682733-28604) [n:127.0.0.1:37281_solr c:testErrorChecks-collection s:shard2 r:core_node4 x:testErrorChecks-collection_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 5336835 INFO (qtp143682733-28604) [n:127.0.0.1:37281_solr c:testErrorChecks-collection s:shard2 r:core_node4 x:testErrorChecks-collection_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 5336835 INFO (qtp143682733-28604) [n:127.0.0.1:37281_solr c:testErrorChecks-collection s:shard2 r:core_node4 x:testErrorChecks-collection_shard2_replica_n2] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:37281/solr/testErrorChecks-collection_shard2_replica_n2/
[junit4] 2> 5336835 INFO (qtp143682733-28604) [n:127.0.0.1:37281_solr c:testErrorChecks-collection s:shard2 r:core_node4 x:testErrorChecks-collection_shard2_replica_n2] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 5336835 INFO (qtp143682733-28604) [n:127.0.0.1:37281_solr c:testErrorChecks-collection s:shard2 r:core_node4 x:testErrorChecks-collection_shard2_replica_n2] o.a.s.c.SyncStrategy https://127.0.0.1:37281/solr/testErrorChecks-collection_shard2_replica_n2/ has no replicas
[junit4] 2> 5336835 INFO (qtp143682733-28604) [n:127.0.0.1:37281_solr c:testErrorChecks-collection s:shard2 r:core_node4 x:testErrorChecks-collection_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 5336838 INFO (qtp143682733-28604) [n:127.0.0.1:37281_solr c:testErrorChecks-collection s:shard2 r:core_node4 x:testErrorChecks-collection_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:37281/solr/testErrorChecks-collection_shard2_replica_n2/ shard2
[junit4] 2> 5336900 INFO (qtp1607421543-28598) [n:127.0.0.1:45225_solr c:testErrorChecks-collection s:shard1 r:core_node3 x:testErrorChecks-collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 5336900 INFO (qtp1607421543-28598) [n:127.0.0.1:45225_solr c:testErrorChecks-collection s:shard1 r:core_node3 x:testErrorChecks-collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 5336901 INFO (qtp1607421543-28598) [n:127.0.0.1:45225_solr c:testErrorChecks-collection s:shard1 r:core_node3 x:testErrorChecks-collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 5336901 INFO (qtp1607421543-28598) [n:127.0.0.1:45225_solr c:testErrorChecks-collection s:shard1 r:core_node3 x:testErrorChecks-collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 5336902 INFO (qtp1607421543-28598) [n:127.0.0.1:45225_solr c:testErrorChecks-collection s:shard1 r:core_node3 x:testErrorChecks-collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@1cf1bf34[testErrorChecks-collection_shard1_replica_n1] main]
[junit4] 2> 5336904 INFO (qtp1607421543-28598) [n:127.0.0.1:45225_solr c:testErrorChecks-collection s:shard1 r:core_node3 x:testErrorChecks-collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 5336904 INFO (qtp1607421543-28598) [n:127.0.0.1:45225_solr c:testErrorChecks-collection s:shard1 r:core_node3 x:testErrorChecks-collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 5336905 INFO (qtp1607421543-28598) [n:127.0.0.1:45225_solr c:testErrorChecks-collection s:shard1 r:core_node3 x:testErrorChecks-collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 5336905 INFO (qtp1607421543-28598) [n:127.0.0.1:45225_solr c:testErrorChecks-collection s:shard1 r:core_node3 x:testErrorChecks-collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1595205739381522432
[junit4] 2> 5336916 INFO (searcherExecutor-8028-thread-1-processing-n:127.0.0.1:45225_solr x:testErrorChecks-collection_shard1_replica_n1 s:shard1 c:testErrorChecks-collection r:core_node3) [n:127.0.0.1:45225_solr c:testErrorChecks-collection s:shard1 r:core_node3 x:testErrorChecks-collection_shard1_replica_n1] o.a.s.c.SolrCore [testErrorChecks-collection_shard1_replica_n1] Registered new searcher Searcher@1cf1bf34[testErrorChecks-collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 5336921 INFO (qtp1607421543-28598) [n:127.0.0.1:45225_solr c:testErrorChecks-collection s:shard1 r:core_node3 x:testErrorChecks-collection_shard1_replica_n1] o.a.s.c.ZkShardTerms Successful update of terms at /collections/testErrorChecks-collection/terms/shard1 to Terms{values={core_node3=0}, version=0}
[junit4] 2> 5336922 INFO (qtp1607421543-28598) [n:127.0.0.1:45225_solr c:testErrorChecks-collection s:shard1 r:core_node3 x:testErrorChecks-collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 5336922 INFO (qtp1607421543-28598) [n:127.0.0.1:45225_solr c:testErrorChecks-collection s:shard1 r:core_node3 x:testErrorChecks-collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 5336922 INFO (qtp1607421543-28598) [n:127.0.0.1:45225_solr c:testErrorChecks-collection s:shard1 r:core_node3 x:testErrorChecks-collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:45225/solr/testErrorChecks-collection_shard1_replica_n1/
[junit4] 2> 5336922 INFO (qtp1607421543-28598) [n:127.0.0.1:45225_solr c:testErrorChecks-collection s:shard1 r:core_node3 x:testErrorChecks-collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 5336922 INFO (qtp1607421543-28598) [n:127.0.0.1:45225_solr c:testErrorChecks-collection s:shard1 r:core_node3 x:testErrorChecks-collection_shard1_replica_n1] o.a.s.c.SyncStrategy https://127.0.0.1:45225/solr/testErrorChecks-collection_shard1_replica_n1/ has no replicas
[junit4] 2> 5336923 INFO (qtp1607421543-28598) [n:127.0.0.1:45225_solr c:testErrorChecks-collection s:shard1 r:core_node3 x:testErrorChecks-collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 5336941 INFO (qtp1607421543-28598) [n:127.0.0.1:45225_solr c:testErrorChecks-collection s:shard1 r:core_node3 x:testErrorChecks-collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:45225/solr/testErrorChecks-collection_shard1_replica_n1/ shard1
[junit4] 2> 5337042 INFO (zkCallback-6081-thread-2-processing-n:127.0.0.1:37281_solr) [n:127.0.0.1:37281_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testErrorChecks-collection/state.json] for collection [testErrorChecks-collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 5337043 INFO (zkCallback-6087-thread-1-processing-n:127.0.0.1:45225_solr) [n:127.0.0.1:45225_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testErrorChecks-collection/state.json] for collection [testErrorChecks-collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 5337043 INFO (zkCallback-6081-thread-1-processing-n:127.0.0.1:37281_solr) [n:127.0.0.1:37281_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testErrorChecks-collection/state.json] for collection [testErrorChecks-collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 5337042 INFO (zkCallback-6087-thread-2-processing-n:127.0.0.1:45225_solr) [n:127.0.0.1:45225_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testErrorChecks-collection/state.json] for collection [testErrorChecks-collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 5337090 INFO (qtp143682733-28604) [n:127.0.0.1:37281_solr c:testErrorChecks-collection s:shard2 r:core_node4 x:testErrorChecks-collection_shard2_replica_n2] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 5337092 INFO (qtp1607421543-28598) [n:127.0.0.1:45225_solr c:testErrorChecks-collection s:shard1 r:core_node3 x:testErrorChecks-collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 5337092 INFO (qtp143682733-28604) [n:127.0.0.1:37281_solr c:testErrorChecks-collection s:shard2 r:core_node4 x:testErrorChecks-collection_shard2_replica_n2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=testErrorChecks-collection_shard2_replica_n2&action=CREATE&numShards=2&collection=testErrorChecks-collection&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2084
[junit4] 2> 5337093 INFO (qtp1607421543-28598) [n:127.0.0.1:45225_solr c:testErrorChecks-collection s:shard1 r:core_node3 x:testErrorChecks-collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=testErrorChecks-collection_shard1_replica_n1&action=CREATE&numShards=2&collection=testErrorChecks-collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2020
[junit4] 2> 5337100 INFO (qtp143682733-28597) [n:127.0.0.1:37281_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 5337194 INFO (zkCallback-6081-thread-2-processing-n:127.0.0.1:37281_solr) [n:127.0.0.1:37281_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testErrorChecks-collection/state.json] for collection [testErrorChecks-collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 5337194 INFO (zkCallback-6081-thread-1-processing-n:127.0.0.1:37281_solr) [n:127.0.0.1:37281_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testErrorChecks-collection/state.json] for collection [testErrorChecks-collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 5337194 INFO (zkCallback-6087-thread-2-processing-n:127.0.0.1:45225_solr) [n:127.0.0.1:45225_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testErrorChecks-collection/state.json] for collection [testErrorChecks-collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 5337194 INFO (zkCallback-6087-thread-1-processing-n:127.0.0.1:45225_solr) [n:127.0.0.1:45225_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testErrorChecks-collection/state.json] for collection [testErrorChecks-collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 5338100 INFO (qtp143682733-28597) [n:127.0.0.1:37281_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf&name=testErrorChecks-collection&nrtReplicas=1&action=CREATE&numShards=2&wt=javabin&version=2} status=0 QTime=3689
[junit4] 2> 5338103 INFO (qtp143682733-28602) [n:127.0.0.1:37281_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :createalias with params collections=doesnotexist&name=testalias&action=CREATEALIAS&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 5338105 INFO (OverseerCollectionConfigSetProcessor-72296416233914374-127.0.0.1:45225_solr-n_0000000000) [n:127.0.0.1:45225_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 5338105 INFO (OverseerThreadFactory-8022-thread-2-processing-n:127.0.0.1:45225_solr) [n:127.0.0.1:45225_solr ] o.a.s.c.a.c.OverseerCollectionMessageHandler Ignoring exception matching .
[junit4] 2> 5338124 INFO (qtp143682733-28602) [n:127.0.0.1:37281_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collections=doesnotexist&name=testalias&action=CREATEALIAS&wt=javabin&version=2} status=400 QTime=20
[junit4] 2> 5338125 INFO (qtp143682733-28597) [n:127.0.0.1:37281_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :createalias with params collections=testErrorChecks-collection,doesnotexist&name=testalias&action=CREATEALIAS&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 5338126 INFO (OverseerThreadFactory-8022-thread-3-processing-n:127.0.0.1:45225_solr) [n:127.0.0.1:45225_solr ] o.a.s.c.a.c.OverseerCollectionMessageHandler Ignoring exception matching .
[junit4] 2> 5338126 INFO (OverseerCollectionConfigSetProcessor-72296416233914374-127.0.0.1:45225_solr-n_0000000000) [n:127.0.0.1:45225_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 5338134 INFO (qtp143682733-28597) [n:127.0.0.1:37281_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collections=testErrorChecks-collection,doesnotexist&name=testalias&action=CREATEALIAS&wt=javabin&version=2} status=400 QTime=8
[junit4] 2> 5338134 INFO (qtp143682733-28602) [n:127.0.0.1:37281_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :createalias with params collections=testErrorChecks-collection&name=testalias&action=CREATEALIAS&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 5338255 INFO (qtp143682733-28602) [n:127.0.0.1:37281_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collections=testErrorChecks-collection&name=testalias&action=CREATEALIAS&wt=javabin&version=2} status=0 QTime=120
[junit4] 2> 5338255 INFO (qtp143682733-28597) [n:127.0.0.1:37281_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :createalias with params collections=testalias&name=testalias2&action=CREATEALIAS&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 5338257 INFO (OverseerCollectionConfigSetProcessor-72296416233914374-127.0.0.1:45225_solr-n_0000000000) [n:127.0.0.1:45225_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000006 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 5338359 INFO (qtp143682733-28597) [n:127.0.0.1:37281_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collections=testalias&name=testalias2&action=CREATEALIAS&wt=javabin&version=2} status=0 QTime=103
[junit4] 2> 5338360 INFO (qtp143682733-28602) [n:127.0.0.1:37281_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :createalias with params collections=testalias2,doesnotexist&name=testalias3&action=CREATEALIAS&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 5338362 INFO (OverseerThreadFactory-8022-thread-5-processing-n:127.0.0.1:45225_solr) [n:127.0.0.1:45225_solr ] o.a.s.c.a.c.OverseerCollectionMessageHandler Ignoring exception matching .
[junit4] 2> 5338362 INFO (OverseerCollectionConfigSetProcessor-72296416233914374-127.0.0.1:45225_solr-n_0000000000) [n:127.0.0.1:45225_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000008 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 5338364 INFO (qtp143682733-28602) [n:127.0.0.1:37281_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collections=testalias2,doesnotexist&name=testalias3&action=CREATEALIAS&wt=javabin&version=2} status=400 QTime=3
[junit4] 2> 5338364 INFO (qtp143682733-28597) [n:127.0.0.1:37281_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :deletealias with params name=testalias&action=DELETEALIAS&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 5338374 INFO (qtp143682733-28597) [n:127.0.0.1:37281_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={name=testalias&action=DELETEALIAS&wt=javabin&version=2} status=0 QTime=10
[junit4] 2> 5338375 INFO (qtp143682733-28602) [n:127.0.0.1:37281_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :deletealias with params name=testalias2&action=DELETEALIAS&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 5338377 INFO (OverseerCollectionConfigSetProcessor-72296416233914374-127.0.0.1:45225_solr-n_0000000000) [n:127.0.0.1:45225_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000012 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 5338378 INFO (qtp143682733-28602) [n:127.0.0.1:37281_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={name=testalias2&action=DELETEALIAS&wt=javabin&version=2} status=0 QTime=3
[junit4] 2> 5338378 INFO (TEST-AliasIntegrationTest.testErrorChecks-seed#[EB534B6BF3D0DA43]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testErrorChecks
[junit4] 2> 5338380 INFO (qtp143682733-28597) [n:127.0.0.1:37281_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :listaliases with params action=LISTALIASES&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 5338380 INFO (qtp143682733-28597) [n:127.0.0.1:37281_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LISTALIASES&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 5338389 INFO (TEST-AliasIntegrationTest.testErrorChecks-seed#[EB534B6BF3D0DA43]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 5338390 INFO (qtp143682733-28602) [n:127.0.0.1:37281_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :delete with params name=testErrorChecks-collection&action=DELETE&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 5338391 INFO (OverseerThreadFactory-8022-thread-5-processing-n:127.0.0.1:45225_solr) [n:127.0.0.1:45225_solr ] o.a.s.c.a.c.OverseerCollectionMessageHandler Executing Collection Cmd=action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true, asyncId=null
[junit4] 2> 5338391 INFO (OverseerCollectionConfigSetProcessor-72296416233914374-127.0.0.1:45225_solr-n_0000000000) [n:127.0.0.1:45225_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000014 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 5338392 INFO (qtp143682733-28603) [n:127.0.0.1:37281_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.testErrorChecks-collection.shard2.replica_n2, tag=null
[junit4] 2> 5338392 INFO (qtp1607421543-28595) [n:127.0.0.1:45225_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.testErrorChecks-collection.shard1.replica_n1, tag=null
[junit4] 2> 5338392 INFO (qtp143682733-28603) [n:127.0.0.1:37281_solr ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@bc0b4dc: rootName = solr_37281, domain = solr.core.testErrorChecks-collection.shard2.replica_n2, service url = null, agent id = null] for registry solr.core.testErrorChecks-collection.shard2.replica_n2 / com.codahale.metrics.MetricRegistry@7e2014f8
[junit4] 2> 5338392 INFO (qtp1607421543-28595) [n:127.0.0.1:45225_solr ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@41fa8ce0: rootName = solr_45225, domain = solr.core.testErrorChecks-collection.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.testErrorChecks-collection.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@60535cc1
[junit4] 2> 5338452 INFO (qtp1607421543-28595) [n:127.0.0.1:45225_solr ] o.a.s.c.SolrCore [testErrorChecks-collection_shard1_replica_n1] CLOSING SolrCore org.apache.solr.core.SolrCore@27d68ce
[junit4] 2> 5338452 INFO (qtp1607421543-28595) [n:127.0.0.1:45225_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.testErrorChecks-collection.shard1.replica_n1, tag=41773262
[junit4] 2> 5338452 INFO (qtp1607421543-28595) [n:127.0.0.1:45225_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.testErrorChecks-collection.shard1.leader, tag=41773262
[junit4] 2> 5338453 INFO (qtp143682733-28603) [n:127.0.0.1:37281_solr ] o.a.s.c.SolrCore [testErrorChecks-collection_shard2_replica_n2] CLOSING SolrCore org.apache.solr.core.SolrCore@2a8cda55
[junit4] 2> 5338453 INFO (qtp143682733-28603) [n:127.0.0.1:37281_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.testErrorChecks-collection.shard2.replica_n2, tag=713874005
[junit4] 2> 5338453 INFO (qtp143682733-28603) [n:127.0.0.1:37281_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.testErrorChecks-collection.shard2.leader, tag=713874005
[junit4] 2> 5338468 INFO (qtp1607421543-28595) [n:127.0.0.1:45225_solr ] o.a.s.c.ZkShardTerms Successful update of terms at /collections/testErrorChecks-collection/terms/shard1 to Terms{values={}, version=1}
[junit4] 2> 5338482 INFO (qtp1607421543-28595) [n:127.0.0.1:45225_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={deleteInstanceDir=true&core=testErrorChecks-collection_shard1_replica_n1&qt=/admin/cores&deleteDataDir=true&action=UNLOAD&wt=javabin&version=2} status=0 QTime=90
[junit4] 2> 5338494 INFO (qtp143682733-28603) [n:127.0.0.1:37281_solr ] o.a.s.c.ZkShardTerms Successful update of terms at /collections/testErrorChecks-collection/terms/shard2 to Terms{values={}, version=1}
[junit4] 2> 5338498 INFO (qtp143682733-28603) [n:127.0.0.1:37281_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={deleteInstanceDir=true&core=testErrorChecks-collection_shard2_replica_n2&qt=/admin/cores&deleteDataDir=true&action=UNLOAD&wt=javabin&version=2} status=0 QTime=106
[junit4] 2> 5339254 INFO (qtp143682733-28602) [n:127.0.0.1:37281_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={name=testErrorChecks-collection&action=DELETE&wt=javabin&version=2} status=0 QTime=864
[junit4] IGNOR/A 0.00s J1 | AliasIntegrationTest.testProperties
[junit4] > Assumption #1: 'badapple' test group is disabled (@BadApple(bugUrl=https://issues.apache.org/jira/browse/SOLR-12028))
[junit4] 2> 5339334 INFO (TEST-AliasIntegrationTest.testModifyPropertiesV1-seed#[EB534B6BF3D0DA43]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testModifyPropertiesV1
[junit4] 2> 5339336 INFO (qtp143682733-28597) [n:127.0.0.1:37281_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf&name=collection1meta&nrtReplicas=1&action=CREATE&numShards=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 5339338 INFO (OverseerThreadFactory-8022-thread-5-processing-n:127.0.0.1:45225_solr) [n:127.0.0.1:45225_solr ] o.a.s.c.a.c.CreateCollectionCmd Create collection collection1meta
[junit4] 2> 5339338 INFO (OverseerCollectionConfigSetProcessor-72296416233914374-127.0.0.1:45225_solr-n_0000000000) [n:127.0.0.1:45225_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000016 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 5339545 INFO (OverseerStateUpdate-72296416233914374-127.0.0.1:45225_solr-n_0000000000) [n:127.0.0.1:45225_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"collection1meta",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"collection1meta_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:37281/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 5339547 INFO (OverseerStateUpdate-72296416233914374-127.0.0.1:45225_solr-n_0000000000) [n:127.0.0.1:45225_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"collection1meta",
[junit4] 2> "shard":"shard2",
[junit4] 2> "core":"collection1meta_shard2_replica_n2",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:45225/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 5339748 INFO (qtp143682733-28604) [n:127.0.0.1:37281_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=collection1meta_shard1_replica_n1&action=CREATE&numShards=2&collection=collection1meta&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 5339749 INFO (qtp1607421543-28598) [n:127.0.0.1:45225_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=collection1meta_shard2_replica_n2&action=CREATE&numShards=2&collection=collection1meta&shard=shard2&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 5339852 INFO (zkCallback-6087-thread-2-processing-n:127.0.0.1:45225_solr) [n:127.0.0.1:45225_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1meta/state.json] for collection [collection1meta] has occurred - updating... (live nodes size: [2])
[junit4] 2> 5339852 INFO (zkCallback-6081-thread-2-processing-n:127.0.0.1:37281_solr) [n:127.0.0.1:37281_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1meta/state.json] for collection [collection1meta] has occurred - updating... (live nodes size: [2])
[junit4] 2> 5339852 INFO (zkCallback-6087-thread-3-processing-n:127.0.0.1:45225_solr) [n:127.0.0.1:45225_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1meta/state.json] for collection [collection1meta] has occurred - updating... (live nodes size: [2])
[junit4] 2> 5339852 INFO (zkCallback-6081-thread-1-processing-n:127.0.0.1:37281_solr) [n:127.0.0.1:37281_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1meta/state.json] for collection [collection1meta] has occurred - updating... (live nodes size: [2])
[junit4] 2> 5340831 INFO (qtp143682733-28604) [n:127.0.0.1:37281_solr c:collection1meta s:shard1 r:core_node3 x:collection1meta_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 5340836 INFO (qtp1607421543-28598) [n:127.0.0.1:45225_solr c:collection1meta s:shard2 r:core_node4 x:collection1meta_shard2_replica_n2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 5340858 INFO (qtp1607421543-28598) [n:127.0.0.1:45225_solr c:collection1meta s:shard2 r:core_node4 x:collection1meta_shard2_replica_n2] o.a.s.s.IndexSchema [collection1meta_shard2_replica_n2] Schema name=minimal
[junit4] 2> 5341100 INFO (qtp1607421543-28598) [n:127.0.0.1:45225_solr c:collection1meta s:shard2 r:core_node4 x:collection1meta_shard2_replica_n2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 5341100 INFO (qtp1607421543-28598) [n:127.0.0.1:45225_solr c:collection1meta s:shard2 r:core_node4 x:collection1meta_shard2_replica_n2] o.a.s.c.CoreContainer Creating SolrCore 'collection1meta_shard2_replica_n2' using configuration from collection collection1meta, trusted=true
[junit4] 2> 5341101 INFO (qtp1607421543-28598) [n:127.0.0.1:45225_solr c:collection1meta s:shard2 r:core_node4 x:collection1meta_shard2_replica_n2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45225.solr.core.collection1meta.shard2.replica_n2' (registry 'solr.core.collection1meta.shard2.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@49922e4f
[junit4] 2> 5341101 INFO (qtp1607421543-28598) [n:127.0.0.1:45225_solr c:collection1meta s:shard2 r:core_node4 x:collection1meta_shard2_replica_n2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 5341101 INFO (qtp1607421543-28598) [n:127.0.0.1:45225_solr c:collection1meta s:shard2 r:core_node4 x:collection1meta_shard2_replica_n2] o.a.s.c.SolrCore [[collection1meta_shard2_replica_n2] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.AliasIntegrationTest_EB534B6BF3D0DA43-001/tempDir-001/node2/collection1meta_shard2_replica_n2], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.AliasIntegrationTest_EB534B6BF3D0DA43-001/tempDir-001/node2/./collection1meta_shard2_replica_n2/data/]
[junit4] 2> 5341117 INFO (qtp143682733-28604) [n:127.0.0.1:37281_solr c:collection1meta s:shard1 r:core_node3 x:collection1meta_shard1_replica_n1] o.a.s.s.IndexSchema [collection1meta_shard1_replica_n1] Schema name=minimal
[junit4] 2> 5341153 INFO (qtp143682733-28604) [n:127.0.0.1:37281_solr c:collection1meta s:shard1 r:core_node3 x:collection1meta_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 5341153 INFO (qtp143682733-28604) [n:127.0.0.1:37281_solr c:collection1meta s:shard1 r:core_node3 x:collection1meta_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'collection1meta_shard1_replica_n1' using configuration from collection collection1meta, trusted=true
[junit4] 2> 5341153 INFO (qtp143682733-28604) [n:127.0.0.1:37281_solr c:collection1meta s:shard1 r:core_node3 x:collection1meta_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_37281.solr.core.collection1meta.shard1.replica_n1' (registry 'solr.core.collection1meta.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@49922e4f
[junit4] 2> 5341154 INFO (qtp143682733-28604) [n:127.0.0.1:37281_solr c:collection1meta s:shard1 r:core_node3 x:collection1meta_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 5341154 INFO (qtp143682733-28604) [n:127.0.0.1:37281_solr c:collection1meta s:shard1 r:core_node3 x:collection1meta_shard1_replica_n1] o.a.s.c.SolrCore [[collection1meta_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.AliasIntegrationTest_EB534B6BF3D0DA43-001/tempDir-001/node1/collection1meta_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.AliasIntegrationTest_EB534B6BF3D0DA43-001/tempDir-001/node1/./collection1meta_shard1_replica_n1/data/]
[junit4] 2> 5341465 INFO (qtp1607421543-28598) [n:127.0.0.1:45225_solr c:collection1meta s:shard2 r:core_node4 x:collection1meta_shard2_replica_n2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 5341465 INFO (qtp1607421543-28598) [n:127.0.0.1:45225_solr c:collection1meta s:shard2 r:core_node4 x:collection1meta_shard2_replica_n2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 5341466 INFO (qtp1607421543-28598) [n:127.0.0.1:45225_solr c:collection1meta s:shard2 r:core_node4 x:collection1meta_shard2_replica_n2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 5341466 INFO (qtp1607421543-28598) [n:127.0.0.1:45225_solr c:collection1meta s:shard2 r:core_node4 x:collection1meta_shard2_replica_n2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 5341467 INFO (qtp1607421543-28598) [n:127.0.0.1:45225_solr c:collection1meta s:shard2 r:core_node4 x:collection1meta_shard2_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@23a4d0cf[collection1meta_shard2_replica_n2] main]
[junit4] 2> 5341483 INFO (qtp1607421543-28598) [n:127.0.0.1:45225_solr c:collection1meta s:shard2 r:core_node4 x:collection1meta_shard2_replica_n2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 5341483 INFO (qtp1607421543-28598) [n:127.0.0.1:45225_solr c:collection1meta s:shard2 r:core_node4 x:collection1meta_shard2_replica_n2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 5341483 INFO (qtp1607421543-28598) [n:127.0.0.1:45225_solr c:collection1meta s:shard2 r:core_node4 x:collection1meta_shard2_replica_n2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 5341484 INFO (searcherExecutor-8037-thread-1-processing-n:127.0.0.1:45225_solr x:collection1meta_shard2_replica_n2 s:shard2 c:collection1meta r:core_node4) [n:127.0.0.1:45225_solr c:collection1meta s:shard2 r:core_node4 x:collection1meta_shard2_replica_n2] o.a.s.c.SolrCore [collection1meta_shard2_replica_n2] Registered new searcher Searcher@23a4d0cf[collection1meta_shard2_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 5341484 INFO (qtp1607421543-28598) [n:127.0.0.1:45225_solr c:collection1meta s:shard2 r:core_node4 x:collection1meta_shard2_replica_n2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1595205744182951936
[junit4] 2> 5341518 INFO (qtp1607421543-28598) [n:127.0.0.1:45225_solr c:collection1meta s:shard2 r:core_node4 x:collection1meta_shard2_replica_n2] o.a.s.c.ZkShardTerms Successful update of terms at /collections/collection1meta/terms/shard2 to Terms{values={core_node4=0}, version=0}
[junit4] 2> 5341520 INFO (qtp1607421543-28598) [n:127.0.0.1:45225_solr c:collection1meta s:shard2 r:core_node4 x:collection1meta_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 5341520 INFO (qtp1607421543-285
[...truncated too long message...]
] > at org.apache.solr.update.TestInPlaceUpdatesDistrib.test(TestInPlaceUpdatesDistrib.java:144)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 8433576 INFO (SUITE-TestInPlaceUpdatesDistrib-seed#[EB534B6BF3D0DA43]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> 8433576 INFO (SUITE-TestInPlaceUpdatesDistrib-seed#[EB534B6BF3D0DA43]-worker) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=321911031
[junit4] 2> 8433576 INFO (SUITE-TestInPlaceUpdatesDistrib-seed#[EB534B6BF3D0DA43]-worker) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 8433576 INFO (SUITE-TestInPlaceUpdatesDistrib-seed#[EB534B6BF3D0DA43]-worker) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@9e88b89: rootName = null, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@1e717997
[junit4] 2> 8433577 INFO (SUITE-TestInPlaceUpdatesDistrib-seed#[EB534B6BF3D0DA43]-worker) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 8433577 INFO (SUITE-TestInPlaceUpdatesDistrib-seed#[EB534B6BF3D0DA43]-worker) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@7a8b570e: rootName = null, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@54600355
[junit4] 2> 8433577 INFO (SUITE-TestInPlaceUpdatesDistrib-seed#[EB534B6BF3D0DA43]-worker) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 8433577 INFO (SUITE-TestInPlaceUpdatesDistrib-seed#[EB534B6BF3D0DA43]-worker) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@fc89177: rootName = null, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@72026be0
[junit4] 2> 8433582 INFO (coreCloseExecutor-14251-thread-1) [ x:collection1] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@5ef6b82e
[junit4] 2> 8433582 INFO (coreCloseExecutor-14251-thread-1) [ x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=1593227310
[junit4] 2> 8433582 INFO (coreCloseExecutor-14251-thread-1) [ x:collection1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@1cef574f: rootName = null, domain = solr.core.collection1, service url = null, agent id = null] for registry solr.core.collection1 / com.codahale.metrics.MetricRegistry@743cca0f
[junit4] 2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.update.TestInPlaceUpdatesDistrib_EB534B6BF3D0DA43-001
[junit4] 2> Mar 17, 2018 6:04:01 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Lucene70, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@d67425c), locale=th-TH-u-nu-thai-x-lvariant-TH, timezone=Asia/Choibalsan
[junit4] 2> NOTE: Linux 4.4.0-112-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=1,free=210378776,total=533200896
[junit4] 2> NOTE: All tests run in this JVM: [TestMiniSolrCloudClusterSSL, TestLegacyFieldCache, MigrateRouteKeyTest, TestJoin, TestLocalFSCloudBackupRestore, TestClassicSimilarityFactory, TestSolrQueryParser, CleanupOldIndexTest, TokenizerChainTest, SuggestComponentContextFilterQueryTest, TestUninvertingReader, TestDynamicLoading, TestUnifiedSolrHighlighter, CursorPagingTest, DistribJoinFromCollectionTest, TestInitParams, ZkStateWriterTest, TestJsonRequest, DistributedTermsComponentTest, MoveReplicaHDFSTest, TestReRankQParserPlugin, LIRRollingUpdatesTest, TestDeleteCollectionOnDownNodes, ConnectionReuseTest, TestFieldCacheSortRandom, MetricUtilsTest, ChaosMonkeyNothingIsSafeTest, TestConfig, TestMissingGroups, TestLegacyFieldReuse, TestDistribIDF, TestHdfsUpdateLog, NodeLostTriggerTest, PKIAuthenticationIntegrationTest, TestImpersonationWithHadoopAuth, CollectionReloadTest, UpdateLogTest, BinaryUpdateRequestHandlerTest, PluginInfoTest, TestCSVResponseWriter, DistributedQueryElevationComponentTest, DocumentAnalysisRequestHandlerTest, HdfsRestartWhileUpdatingTest, TestNumericTokenStream, TestMacros, QueryResultKeyTest, OutputWriterTest, BooleanFieldTest, SuggesterWFSTTest, TestMaxTokenLenTokenizer, TestPolicyCloud, TestPseudoReturnFields, InfoHandlerTest, TestSolrDeletionPolicy1, RecoveryAfterSoftCommitTest, TestManagedSynonymFilterFactory, TestInitQParser, TestNestedDocsSort, PreAnalyzedUpdateProcessorTest, TestPushWriter, SolrGraphiteReporterTest, DocValuesNotIndexedTest, TestEmbeddedSolrServerConstructors, TestHighlightDedupGrouping, TestUseDocValuesAsStored2, BasicDistributedZkTest, HdfsDirectoryTest, DistributedSuggestComponentTest, TestSearcherReuse, TestNamedUpdateProcessors, OverseerCollectionConfigSetProcessorTest, TestExtendedDismaxParser, HttpPartitionOnCommitTest, TestAddFieldRealTimeGet, TestLuceneMatchVersion, TestMultiValuedNumericRangeQuery, ZkCLITest, RegexBytesRefFilterTest, LeaderFailureAfterFreshStartTest, HdfsBasicDistributedZkTest, TestDynamicFieldResource, CdcrVersionReplicationTest, TestConfigSetsAPIExclusivity, BlockCacheTest, CachingDirectoryFactoryTest, UpdateParamsTest, TestSolrXml, TestRemoteStreaming, TestGroupingSearch, TestMergePolicyConfig, ComputePlanActionTest, SolrPluginUtilsTest, FieldAnalysisRequestHandlerTest, BlockJoinFacetSimpleTest, SpellingQueryConverterTest, FullSolrCloudDistribCmdsTest, TestDistributedSearch, TestSolrCLIRunExample, ReplicaListTransformerTest, ZkNodePropsTest, TestStandardQParsers, DistributedExpandComponentTest, TestSSLRandomization, NodeMutatorTest, HighlighterConfigTest, EchoParamsTest, DirectoryFactoryTest, BJQParserTest, SecurityConfHandlerTest, TestCollationField, StatsComponentTest, OverseerStatusTest, RAMDirectoryFactoryTest, ExternalFileFieldSortTest, TestXmlQParser, ConfigSetsAPITest, TestStressLucene, DebugComponentTest, TestTrackingShardHandlerFactory, TestDocumentBuilder, TlogReplayBufferedWhileIndexingTest, TestOverriddenPrefixQueryForCustomFieldType, DistributedFacetPivotWhiteBoxTest, SolrIndexConfigTest, DeleteStatusTest, DeleteNodeTest, TestElisionMultitermQuery, TestPhraseSuggestions, TestCodecSupport, SystemInfoHandlerTest, FileUtilsTest, DistributedMLTComponentTest, TestEmbeddedSolrServerAdminHandler, AddReplicaTest, ChaosMonkeyNothingIsSafeWithPullReplicasTest, DistributedVersionInfoTest, LeaderFailoverAfterPartitionTest, MoveReplicaHDFSFailoverTest, SaslZkACLProviderTest, TestAuthenticationFramework, TestCloudConsistency, TestCloudSearcherWarming, TestDistribDocBasedVersion, TestLeaderInitiatedRecoveryThread, TestPullReplicaErrorHandling, TestRandomFlRTGCloud, TestSolrCloudWithKerberosAlt, TestStressCloudBlindAtomicUpdates, TestStressLiveNodes, TestCollectionAPI, TestCollectionsAPIViaSolrCloudCluster, TestReplicaProperties, NodeAddedTriggerTest, TriggerIntegrationTest, BlobRepositoryCloudTest, CoreSorterTest, HdfsDirectoryFactoryTest, TestCorePropertiesReload, TestInfoStreamLogging, TestNRTOpen, TestReloadAndDeleteDocs, TestShardHandlerFactory, TestSimpleTextCodec, TestSolrConfigHandler, RequestLoggingTest, SearchHandlerTest, TestCoreBackup, TestHdfsBackupRestoreCore, TestSystemCollAutoCreate, V2ApiIntegrationTest, DistributedFacetPivotLongTailTest, DistributedFacetPivotSmallAdvancedTest, MetricsConfigTest, SolrMetricReporterTest, SolrGangliaReporterTest, SolrJmxReporterTest, TestMacroExpander, JSONWriterTest, SmileWriterTest, TestBinaryResponseWriter, TestCustomDocTransformer, TestExportWriter, TestGeoJSONResponseWriter, TestRawTransformer, TestSubQueryTransformer, TestSubQueryTransformerDistrib, TestFieldCollectionResource, TestFieldResource, TestFieldTypeCollectionResource, TestSchemaNameResource, TestSchemaSimilarityResource, TestManagedStopFilterFactory, DateRangeFieldTest, PreAnalyzedFieldManagedSchemaCloudTest, SchemaApiFailureTest, TestCloudManagedSchema, TestCollationFieldDocValues, TestHalfAndHalfDocValues, TestManagedSchemaThreadSafety, TestPointFields, TestSchemalessBufferedUpdates, AnalyticsMergeStrategyTest, RankQueryTest, TestCollapseQParserPlugin, TestLegacyNumericRangeQueryBuilder, TestMaxScoreQueryParser, TestPayloadScoreQParserPlugin, TestRecoveryHdfs, BasicAuthStandaloneTest, TestAuthorizationFramework, SpellCheckCollatorWithCollapseTest, TestFreeTextSuggestions, BlockDirectoryTest, TestNumericTerms32, CdcrUpdateLogTest, PeerSyncWithIndexFingerprintCachingTest, SolrIndexMetricsTest, TestInPlaceUpdatesDistrib]
[junit4] Completed [784/785 (4!)] on J0 in 238.00s, 1 test, 1 error <<< FAILURES!
[...truncated 9271 lines...]
BUILD FAILED
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:636: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:570: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid13381.hprof
Total time: 558 minutes 26 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1503 - Still
unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1503/
8 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.analytics.facet.QueryFacetTest
Error Message:
Error from server at https://127.0.0.1:33906/solr: Expected mime type application/octet-stream but got text/html. <h1>Bad Message 400</h1><pre>reason: Illegal character 0x0</pre>
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:33906/solr: Expected mime type application/octet-stream but got text/html. <h1>Bad Message 400</h1><pre>reason: Illegal character 0x0</pre>
at __randomizedtesting.SeedInfo.seed([6005039964E31AA5]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:607)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1105)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:885)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:818)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
at org.apache.solr.analytics.SolrAnalyticsTestCase.setupCollection(SolrAnalyticsTestCase.java:62)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:874)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TestLeaderElectionZkExpiry
Error Message:
ObjectTracker found 1 object(s) that were not released!!! [Overseer] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.Overseer at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.cloud.Overseer.start(Overseer.java:545) at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:850) at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170) at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135) at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307) at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:216) at org.apache.solr.cloud.ZkController$1.command(ZkController.java:355) at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:167) at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57) at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:141) at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:531) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506)
Stack Trace:
java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [Overseer]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.Overseer
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.cloud.Overseer.start(Overseer.java:545)
at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:850)
at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)
at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)
at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307)
at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:216)
at org.apache.solr.cloud.ZkController$1.command(ZkController.java:355)
at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:167)
at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57)
at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:141)
at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:531)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506)
at __randomizedtesting.SeedInfo.seed([FE8EF5D38704C33C]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertNull(Assert.java:551)
at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:301)
at sun.reflect.GeneratedMethodAccessor51.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:897)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TestLeaderElectionZkExpiry
Error Message:
1 thread leaked from SUITE scope at org.apache.solr.cloud.TestLeaderElectionZkExpiry: 1) Thread[id=52356, name=OverseerAutoScalingTriggerThread-72284390907117579-dummy.host.com:8984_solr-n_0000000001, state=RUNNABLE, group=Overseer autoscaling triggers] at org.apache.lucene.util.TestRuleLimitSysouts$DelegateStream.write(TestRuleLimitSysouts.java:128) at java.io.PrintStream.write(PrintStream.java:480) at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221) at sun.nio.cs.StreamEncoder.implFlushBuffer(StreamEncoder.java:291) at sun.nio.cs.StreamEncoder.implFlush(StreamEncoder.java:295) at sun.nio.cs.StreamEncoder.flush(StreamEncoder.java:141) at java.io.OutputStreamWriter.flush(OutputStreamWriter.java:229) at org.apache.log4j.helpers.QuietWriter.flush(QuietWriter.java:59) at org.apache.log4j.WriterAppender.subAppend(WriterAppender.java:324) at org.apache.log4j.WriterAppender.append(WriterAppender.java:162) at org.apache.log4j.AppenderSkeleton.doAppend(AppenderSkeleton.java:251) at org.apache.log4j.helpers.AppenderAttachableImpl.appendLoopOnAppenders(AppenderAttachableImpl.java:66) at org.apache.log4j.Category.callAppenders(Category.java:206) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.error(Log4jLoggerAdapter.java:576) at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:142) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.TestLeaderElectionZkExpiry:
1) Thread[id=52356, name=OverseerAutoScalingTriggerThread-72284390907117579-dummy.host.com:8984_solr-n_0000000001, state=RUNNABLE, group=Overseer autoscaling triggers]
at org.apache.lucene.util.TestRuleLimitSysouts$DelegateStream.write(TestRuleLimitSysouts.java:128)
at java.io.PrintStream.write(PrintStream.java:480)
at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221)
at sun.nio.cs.StreamEncoder.implFlushBuffer(StreamEncoder.java:291)
at sun.nio.cs.StreamEncoder.implFlush(StreamEncoder.java:295)
at sun.nio.cs.StreamEncoder.flush(StreamEncoder.java:141)
at java.io.OutputStreamWriter.flush(OutputStreamWriter.java:229)
at org.apache.log4j.helpers.QuietWriter.flush(QuietWriter.java:59)
at org.apache.log4j.WriterAppender.subAppend(WriterAppender.java:324)
at org.apache.log4j.WriterAppender.append(WriterAppender.java:162)
at org.apache.log4j.AppenderSkeleton.doAppend(AppenderSkeleton.java:251)
at org.apache.log4j.helpers.AppenderAttachableImpl.appendLoopOnAppenders(AppenderAttachableImpl.java:66)
at org.apache.log4j.Category.callAppenders(Category.java:206)
at org.apache.log4j.Category.forcedLog(Category.java:391)
at org.apache.log4j.Category.log(Category.java:856)
at org.slf4j.impl.Log4jLoggerAdapter.error(Log4jLoggerAdapter.java:576)
at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:142)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([FE8EF5D38704C33C]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TestLeaderElectionZkExpiry
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=52356, name=OverseerAutoScalingTriggerThread-72284390907117579-dummy.host.com:8984_solr-n_0000000001, state=RUNNABLE, group=Overseer autoscaling triggers] at java.security.AccessController.doPrivileged(Native Method) at com.carrotsearch.ant.tasks.junit4.events.Serializer.flushQueue(Serializer.java:114) at com.carrotsearch.ant.tasks.junit4.events.Serializer.serialize(Serializer.java:99) at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain$3$2.write(SlaveMain.java:472) at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) at java.io.PrintStream.flush(PrintStream.java:338) at java.io.FilterOutputStream.flush(FilterOutputStream.java:140) at java.io.PrintStream.write(PrintStream.java:482) at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221) at sun.nio.cs.StreamEncoder.implFlushBuffer(StreamEncoder.java:291) at sun.nio.cs.StreamEncoder.implFlush(StreamEncoder.java:295) at sun.nio.cs.StreamEncoder.flush(StreamEncoder.java:141) at java.io.OutputStreamWriter.flush(OutputStreamWriter.java:229) at org.apache.log4j.helpers.QuietWriter.flush(QuietWriter.java:59) at org.apache.log4j.WriterAppender.subAppend(WriterAppender.java:324) at org.apache.log4j.WriterAppender.append(WriterAppender.java:162) at org.apache.log4j.AppenderSkeleton.doAppend(AppenderSkeleton.java:251) at org.apache.log4j.helpers.AppenderAttachableImpl.appendLoopOnAppenders(AppenderAttachableImpl.java:66) at org.apache.log4j.Category.callAppenders(Category.java:206) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.error(Log4jLoggerAdapter.java:576) at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:142) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=52356, name=OverseerAutoScalingTriggerThread-72284390907117579-dummy.host.com:8984_solr-n_0000000001, state=RUNNABLE, group=Overseer autoscaling triggers]
at java.security.AccessController.doPrivileged(Native Method)
at com.carrotsearch.ant.tasks.junit4.events.Serializer.flushQueue(Serializer.java:114)
at com.carrotsearch.ant.tasks.junit4.events.Serializer.serialize(Serializer.java:99)
at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain$3$2.write(SlaveMain.java:472)
at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)
at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140)
at java.io.PrintStream.flush(PrintStream.java:338)
at java.io.FilterOutputStream.flush(FilterOutputStream.java:140)
at java.io.PrintStream.write(PrintStream.java:482)
at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221)
at sun.nio.cs.StreamEncoder.implFlushBuffer(StreamEncoder.java:291)
at sun.nio.cs.StreamEncoder.implFlush(StreamEncoder.java:295)
at sun.nio.cs.StreamEncoder.flush(StreamEncoder.java:141)
at java.io.OutputStreamWriter.flush(OutputStreamWriter.java:229)
at org.apache.log4j.helpers.QuietWriter.flush(QuietWriter.java:59)
at org.apache.log4j.WriterAppender.subAppend(WriterAppender.java:324)
at org.apache.log4j.WriterAppender.append(WriterAppender.java:162)
at org.apache.log4j.AppenderSkeleton.doAppend(AppenderSkeleton.java:251)
at org.apache.log4j.helpers.AppenderAttachableImpl.appendLoopOnAppenders(AppenderAttachableImpl.java:66)
at org.apache.log4j.Category.callAppenders(Category.java:206)
at org.apache.log4j.Category.forcedLog(Category.java:391)
at org.apache.log4j.Category.log(Category.java:856)
at org.slf4j.impl.Log4jLoggerAdapter.error(Log4jLoggerAdapter.java:576)
at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:142)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([FE8EF5D38704C33C]:0)
FAILED: org.apache.solr.logging.TestLogWatcher.testLog4jWatcher
Error Message:
expected:<21> but was:<1>
Stack Trace:
java.lang.AssertionError: expected:<21> but was:<1>
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:472)
at org.junit.Assert.assertEquals(Assert.java:456)
at org.apache.solr.logging.TestLogWatcher.testLog4jWatcher(TestLogWatcher.java:52)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:45)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:15)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:42)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:20)
at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:28)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:263)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:68)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:47)
at org.junit.runners.ParentRunner$3.run(ParentRunner.java:231)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:60)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:229)
at org.junit.runners.ParentRunner.access$000(ParentRunner.java:50)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:222)
at org.junit.runners.ParentRunner.run(ParentRunner.java:300)
at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:251)
at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
FAILED: org.apache.solr.security.BasicAuthIntegrationTest.testBasicAuth
Error Message:
Could not load collection from ZK: authCollection
Stack Trace:
org.apache.solr.common.SolrException: Could not load collection from ZK: authCollection
at __randomizedtesting.SeedInfo.seed([FE8EF5D38704C33C:42E083C123574046]:0)
at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1232)
at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:670)
at org.apache.solr.common.cloud.ClusterState$CollectionRef.get(ClusterState.java:386)
at org.apache.solr.client.solrj.impl.CloudSolrClient.getDocCollection(CloudSolrClient.java:1207)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:850)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:818)
at org.apache.solr.security.BasicAuthIntegrationTest.testBasicAuth(BasicAuthIntegrationTest.java:202)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/authCollection/state.json
at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1244)
at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1230)
... 45 more
FAILED: org.apache.solr.update.PeerSyncTest.test
Error Message:
.response[0][id][0]:2!=1
Stack Trace:
junit.framework.AssertionFailedError: .response[0][id][0]:2!=1
at __randomizedtesting.SeedInfo.seed([FE8EF5D38704C33C:76DACA0929F8AEC4]:0)
at junit.framework.Assert.fail(Assert.java:50)
at org.apache.solr.BaseDistributedSearchTestCase.compareSolrResponses(BaseDistributedSearchTestCase.java:913)
at org.apache.solr.BaseDistributedSearchTestCase.compareResponses(BaseDistributedSearchTestCase.java:940)
at org.apache.solr.BaseDistributedSearchTestCase.queryAndCompare(BaseDistributedSearchTestCase.java:650)
at org.apache.solr.BaseDistributedSearchTestCase.queryAndCompare(BaseDistributedSearchTestCase.java:641)
at org.apache.solr.update.PeerSyncTest.test(PeerSyncTest.java:105)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.common.cloud.TestCollectionStateWatchers.testWaitForStateWatcherIsRetainedOnPredicateFailure
Error Message:
Did not see a fully active cluster after 30 seconds
Stack Trace:
java.lang.AssertionError: Did not see a fully active cluster after 30 seconds
at __randomizedtesting.SeedInfo.seed([811295EA81EB6161:92437B959448973]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.common.cloud.TestCollectionStateWatchers.testWaitForStateWatcherIsRetainedOnPredicateFailure(TestCollectionStateWatchers.java:250)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 2265 lines...]
[junit4] JVM J0: stdout was not empty, see: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp/junit4-J0-20180315_061211_5739197370911581534763.sysout
[junit4] >>> JVM J0 emitted unexpected output (verbatim) ----
[junit4] codec: Lucene70, pf: Asserting, dvf: Asserting
[junit4] <<< JVM J0: EOF ----
[...truncated 10978 lines...]
[junit4] Suite: org.apache.solr.security.BasicAuthIntegrationTest
[junit4] 2> 347753 INFO (SUITE-BasicAuthIntegrationTest-seed#[FE8EF5D38704C33C]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.security.BasicAuthIntegrationTest_FE8EF5D38704C33C-001/init-core-data-001
[junit4] 2> 347754 WARN (SUITE-BasicAuthIntegrationTest-seed#[FE8EF5D38704C33C]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=266 numCloses=266
[junit4] 2> 347755 INFO (SUITE-BasicAuthIntegrationTest-seed#[FE8EF5D38704C33C]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 347773 INFO (SUITE-BasicAuthIntegrationTest-seed#[FE8EF5D38704C33C]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 347788 INFO (SUITE-BasicAuthIntegrationTest-seed#[FE8EF5D38704C33C]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 3 servers in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.security.BasicAuthIntegrationTest_FE8EF5D38704C33C-001/tempDir-001
[junit4] 2> 347788 INFO (SUITE-BasicAuthIntegrationTest-seed#[FE8EF5D38704C33C]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 347788 INFO (Thread-1155) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 347788 INFO (Thread-1155) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 347790 ERROR (Thread-1155) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 347897 INFO (SUITE-BasicAuthIntegrationTest-seed#[FE8EF5D38704C33C]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:43676
[junit4] 2> 347915 INFO (zkConnectionManagerCallback-173-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 347951 INFO (jetty-launcher-170-thread-2) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T05:27:37+08:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 347952 INFO (jetty-launcher-170-thread-3) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T05:27:37+08:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 347991 INFO (jetty-launcher-170-thread-1) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T05:27:37+08:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 347991 INFO (jetty-launcher-170-thread-3) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 347993 INFO (jetty-launcher-170-thread-2) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 348011 INFO (jetty-launcher-170-thread-3) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 348011 INFO (jetty-launcher-170-thread-2) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 348011 INFO (jetty-launcher-170-thread-3) [ ] o.e.j.s.session Scavenging every 660000ms
[junit4] 2> 348011 INFO (jetty-launcher-170-thread-2) [ ] o.e.j.s.session Scavenging every 660000ms
[junit4] 2> 348011 INFO (jetty-launcher-170-thread-3) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6cc5f3b8{/solr,null,AVAILABLE}
[junit4] 2> 348015 INFO (jetty-launcher-170-thread-3) [ ] o.e.j.s.AbstractConnector Started ServerConnector@32124f25{HTTP/1.1,[http/1.1]}{127.0.0.1:43964}
[junit4] 2> 348016 INFO (jetty-launcher-170-thread-3) [ ] o.e.j.s.Server Started @359887ms
[junit4] 2> 348016 INFO (jetty-launcher-170-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=43964}
[junit4] 2> 348016 ERROR (jetty-launcher-170-thread-3) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 348016 INFO (jetty-launcher-170-thread-3) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 8.0.0
[junit4] 2> 348016 INFO (jetty-launcher-170-thread-3) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 348016 INFO (jetty-launcher-170-thread-3) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 348016 INFO (jetty-launcher-170-thread-3) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-03-15T13:48:18.589Z
[junit4] 2> 348017 INFO (jetty-launcher-170-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@77f44f51{/solr,null,AVAILABLE}
[junit4] 2> 348018 INFO (jetty-launcher-170-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@9bf77ca{HTTP/1.1,[http/1.1]}{127.0.0.1:41028}
[junit4] 2> 348018 INFO (jetty-launcher-170-thread-2) [ ] o.e.j.s.Server Started @359890ms
[junit4] 2> 348018 INFO (jetty-launcher-170-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=41028}
[junit4] 2> 348055 INFO (jetty-launcher-170-thread-1) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 348055 INFO (jetty-launcher-170-thread-1) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 348055 INFO (jetty-launcher-170-thread-1) [ ] o.e.j.s.session Scavenging every 660000ms
[junit4] 2> 348056 INFO (jetty-launcher-170-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5bf29aea{/solr,null,AVAILABLE}
[junit4] 2> 348056 INFO (jetty-launcher-170-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@64385b78{HTTP/1.1,[http/1.1]}{127.0.0.1:38671}
[junit4] 2> 348056 INFO (jetty-launcher-170-thread-1) [ ] o.e.j.s.Server Started @359928ms
[junit4] 2> 348056 INFO (jetty-launcher-170-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=38671}
[junit4] 2> 348044 ERROR (jetty-launcher-170-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 348076 INFO (jetty-launcher-170-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 8.0.0
[junit4] 2> 348076 INFO (jetty-launcher-170-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 348076 INFO (jetty-launcher-170-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 348076 INFO (jetty-launcher-170-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-03-15T13:48:18.649Z
[junit4] 2> 348057 ERROR (jetty-launcher-170-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 348077 INFO (jetty-launcher-170-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 8.0.0
[junit4] 2> 348077 INFO (jetty-launcher-170-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 348077 INFO (jetty-launcher-170-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 348077 INFO (jetty-launcher-170-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-03-15T13:48:18.650Z
[junit4] 2> 348100 INFO (zkConnectionManagerCallback-175-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 348119 INFO (zkConnectionManagerCallback-177-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 348120 INFO (jetty-launcher-170-thread-3) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 348137 INFO (jetty-launcher-170-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 348165 INFO (zkConnectionManagerCallback-179-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 348156 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x100ce2daaa40001, likely client has closed socket
[junit4] 2> 348172 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x100ce2daaa40002, likely client has closed socket
[junit4] 2> 348189 INFO (jetty-launcher-170-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 348190 INFO (jetty-launcher-170-thread-3) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:43676/solr
[junit4] 2> 348255 INFO (jetty-launcher-170-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:43676/solr
[junit4] 2> 348290 INFO (zkConnectionManagerCallback-183-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 348331 INFO (jetty-launcher-170-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:43676/solr
[junit4] 2> 348333 INFO (zkConnectionManagerCallback-187-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 348360 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x100ce2daaa40004, likely client has closed socket
[junit4] 2> 348360 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x100ce2daaa40005, likely client has closed socket
[junit4] 2> 348360 INFO (zkConnectionManagerCallback-191-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 348381 INFO (zkConnectionManagerCallback-195-thread-1-processing-n:127.0.0.1:41028_solr) [n:127.0.0.1:41028_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 348382 INFO (zkConnectionManagerCallback-194-thread-1-processing-n:127.0.0.1:43964_solr) [n:127.0.0.1:43964_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 348489 INFO (zkConnectionManagerCallback-197-thread-1-processing-n:127.0.0.1:38671_solr) [n:127.0.0.1:38671_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 349273 INFO (jetty-launcher-170-thread-3) [n:127.0.0.1:43964_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 349273 INFO (jetty-launcher-170-thread-3) [n:127.0.0.1:43964_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:43964_solr
[junit4] 2> 349274 INFO (jetty-launcher-170-thread-3) [n:127.0.0.1:43964_solr ] o.a.s.c.Overseer Overseer (id=72284289569652744-127.0.0.1:43964_solr-n_0000000000) starting
[junit4] 2> 349412 INFO (zkConnectionManagerCallback-204-thread-1-processing-n:127.0.0.1:43964_solr) [n:127.0.0.1:43964_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 349445 INFO (jetty-launcher-170-thread-3) [n:127.0.0.1:43964_solr ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:43676/solr ready
[junit4] 2> 349465 INFO (jetty-launcher-170-thread-2) [n:127.0.0.1:41028_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 349480 INFO (jetty-launcher-170-thread-3) [n:127.0.0.1:43964_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43964_solr
[junit4] 2> 349636 INFO (zkCallback-203-thread-1-processing-n:127.0.0.1:43964_solr) [n:127.0.0.1:43964_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 349662 INFO (zkCallback-193-thread-1-processing-n:127.0.0.1:41028_solr) [n:127.0.0.1:41028_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 349709 INFO (OverseerStateUpdate-72284289569652744-127.0.0.1:43964_solr-n_0000000000) [n:127.0.0.1:43964_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 349750 INFO (jetty-launcher-170-thread-2) [n:127.0.0.1:41028_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41028_solr
[junit4] 2> 349838 INFO (zkCallback-192-thread-1-processing-n:127.0.0.1:43964_solr) [n:127.0.0.1:43964_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 349846 INFO (zkCallback-203-thread-1-processing-n:127.0.0.1:43964_solr) [n:127.0.0.1:43964_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 349847 INFO (zkCallback-193-thread-1-processing-n:127.0.0.1:41028_solr) [n:127.0.0.1:41028_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 350260 INFO (jetty-launcher-170-thread-2) [n:127.0.0.1:41028_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41028.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1f2b90af
[junit4] 2> 350282 INFO (jetty-launcher-170-thread-3) [n:127.0.0.1:43964_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43964.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1f2b90af
[junit4] 2> 350301 INFO (jetty-launcher-170-thread-3) [n:127.0.0.1:43964_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43964.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1f2b90af
[junit4] 2> 350301 INFO (jetty-launcher-170-thread-3) [n:127.0.0.1:43964_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43964.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1f2b90af
[junit4] 2> 350309 INFO (jetty-launcher-170-thread-1) [n:127.0.0.1:38671_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 350339 INFO (jetty-launcher-170-thread-3) [n:127.0.0.1:43964_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.security.BasicAuthIntegrationTest_FE8EF5D38704C33C-001/tempDir-001/node2/.
[junit4] 2> 350401 INFO (jetty-launcher-170-thread-2) [n:127.0.0.1:41028_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41028.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1f2b90af
[junit4] 2> 350401 INFO (jetty-launcher-170-thread-2) [n:127.0.0.1:41028_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41028.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1f2b90af
[junit4] 2> 350421 INFO (jetty-launcher-170-thread-2) [n:127.0.0.1:41028_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.security.BasicAuthIntegrationTest_FE8EF5D38704C33C-001/tempDir-001/node3/.
[junit4] 2> 350473 INFO (jetty-launcher-170-thread-1) [n:127.0.0.1:38671_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 350475 INFO (jetty-launcher-170-thread-1) [n:127.0.0.1:38671_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38671_solr
[junit4] 2> 350505 INFO (zkCallback-203-thread-1-processing-n:127.0.0.1:43964_solr) [n:127.0.0.1:43964_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 350509 INFO (zkCallback-193-thread-1-processing-n:127.0.0.1:41028_solr) [n:127.0.0.1:41028_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 350643 INFO (zkCallback-192-thread-2-processing-n:127.0.0.1:43964_solr) [n:127.0.0.1:43964_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 350643 INFO (zkConnectionManagerCallback-208-thread-1-processing-n:127.0.0.1:41028_solr) [n:127.0.0.1:41028_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 350686 INFO (jetty-launcher-170-thread-2) [n:127.0.0.1:41028_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 350714 INFO (zkCallback-196-thread-1-processing-n:127.0.0.1:38671_solr) [n:127.0.0.1:38671_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 350715 INFO (jetty-launcher-170-thread-2) [n:127.0.0.1:41028_solr ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:43676/solr ready
[junit4] 2> 350901 INFO (jetty-launcher-170-thread-1) [n:127.0.0.1:38671_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38671.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1f2b90af
[junit4] 2> 350933 INFO (jetty-launcher-170-thread-1) [n:127.0.0.1:38671_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38671.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1f2b90af
[junit4] 2> 350933 INFO (jetty-launcher-170-thread-1) [n:127.0.0.1:38671_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38671.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1f2b90af
[junit4] 2> 350934 INFO (jetty-launcher-170-thread-1) [n:127.0.0.1:38671_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.security.BasicAuthIntegrationTest_FE8EF5D38704C33C-001/tempDir-001/node1/.
[junit4] 2> 350955 INFO (zkConnectionManagerCallback-212-thread-1-processing-n:127.0.0.1:38671_solr) [n:127.0.0.1:38671_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 350972 INFO (jetty-launcher-170-thread-1) [n:127.0.0.1:38671_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 350973 INFO (jetty-launcher-170-thread-1) [n:127.0.0.1:38671_solr ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:43676/solr ready
[junit4] 2> 351079 INFO (zkConnectionManagerCallback-214-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 351107 INFO (zkConnectionManagerCallback-218-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 351108 INFO (SUITE-BasicAuthIntegrationTest-seed#[FE8EF5D38704C33C]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 351109 INFO (SUITE-BasicAuthIntegrationTest-seed#[FE8EF5D38704C33C]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:43676/solr ready
[junit4] 2> 351123 INFO (qtp270627030-1899) [n:127.0.0.1:38671_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf&name=authCollection&nrtReplicas=1&action=CREATE&numShards=3&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 351261 INFO (OverseerThreadFactory-331-thread-1-processing-n:127.0.0.1:43964_solr) [n:127.0.0.1:43964_solr ] o.a.s.c.a.c.CreateCollectionCmd Create collection authCollection
[junit4] 2> 352089 INFO (OverseerStateUpdate-72284289569652744-127.0.0.1:43964_solr-n_0000000000) [n:127.0.0.1:43964_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"authCollection",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"authCollection_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:38671/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 352309 INFO (OverseerStateUpdate-72284289569652744-127.0.0.1:43964_solr-n_0000000000) [n:127.0.0.1:43964_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"authCollection",
[junit4] 2> "shard":"shard2",
[junit4] 2> "core":"authCollection_shard2_replica_n2",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:43964/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 352366 INFO (OverseerStateUpdate-72284289569652744-127.0.0.1:43964_solr-n_0000000000) [n:127.0.0.1:43964_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"authCollection",
[junit4] 2> "shard":"shard3",
[junit4] 2> "core":"authCollection_shard3_replica_n4",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:41028/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 352675 INFO (qtp270627030-1904) [n:127.0.0.1:38671_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=authCollection_shard1_replica_n1&action=CREATE&numShards=3&collection=authCollection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 352675 INFO (qtp270627030-1904) [n:127.0.0.1:38671_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 352747 INFO (qtp1307534678-1883) [n:127.0.0.1:41028_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node6&collection.configName=conf&newCollection=true&name=authCollection_shard3_replica_n4&action=CREATE&numShards=3&collection=authCollection&shard=shard3&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 352748 INFO (qtp1307534678-1883) [n:127.0.0.1:41028_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 352763 INFO (qtp1161232537-1885) [n:127.0.0.1:43964_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node5&collection.configName=conf&newCollection=true&name=authCollection_shard2_replica_n2&action=CREATE&numShards=3&collection=authCollection&shard=shard2&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 352764 INFO (qtp1161232537-1885) [n:127.0.0.1:43964_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 353240 INFO (zkCallback-193-thread-1-processing-n:127.0.0.1:41028_solr) [n:127.0.0.1:41028_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
[junit4] 2> 353240 INFO (zkCallback-196-thread-1-processing-n:127.0.0.1:38671_solr) [n:127.0.0.1:38671_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
[junit4] 2> 353241 INFO (zkCallback-192-thread-2-processing-n:127.0.0.1:43964_solr) [n:127.0.0.1:43964_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
[junit4] 2> 353242 INFO (zkCallback-193-thread-2-processing-n:127.0.0.1:41028_solr) [n:127.0.0.1:41028_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
[junit4] 2> 353251 INFO (zkCallback-192-thread-1-processing-n:127.0.0.1:43964_solr) [n:127.0.0.1:43964_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
[junit4] 2> 353296 INFO (zkCallback-196-thread-2-processing-n:127.0.0.1:38671_solr) [n:127.0.0.1:38671_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
[junit4] 2> 354281 INFO (qtp270627030-1904) [n:127.0.0.1:38671_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 354298 INFO (qtp1307534678-1883) [n:127.0.0.1:41028_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 354355 INFO (qtp1161232537-1885) [n:127.0.0.1:43964_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 358304 INFO (qtp270627030-1904) [n:127.0.0.1:38671_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.s.IndexSchema [authCollection_shard1_replica_n1] Schema name=minimal
[junit4] 2> 358307 INFO (qtp270627030-1904) [n:127.0.0.1:38671_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 358307 INFO (qtp270627030-1904) [n:127.0.0.1:38671_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'authCollection_shard1_replica_n1' using configuration from collection authCollection, trusted=true
[junit4] 2> 358308 INFO (qtp270627030-1904) [n:127.0.0.1:38671_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38671.solr.core.authCollection.shard1.replica_n1' (registry 'solr.core.authCollection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1f2b90af
[junit4] 2> 358342 INFO (qtp1161232537-1885) [n:127.0.0.1:43964_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.s.IndexSchema [authCollection_shard2_replica_n2] Schema name=minimal
[junit4] 2> 358344 INFO (qtp1161232537-1885) [n:127.0.0.1:43964_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 358417 INFO (qtp270627030-1904) [n:127.0.0.1:38671_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 358417 INFO (qtp270627030-1904) [n:127.0.0.1:38671_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.c.SolrCore [[authCollection_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.security.BasicAuthIntegrationTest_FE8EF5D38704C33C-001/tempDir-001/node1/authCollection_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.security.BasicAuthIntegrationTest_FE8EF5D38704C33C-001/tempDir-001/node1/./authCollection_shard1_replica_n1/data/]
[junit4] 2> 358434 INFO (qtp1161232537-1885) [n:127.0.0.1:43964_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.c.CoreContainer Creating SolrCore 'authCollection_shard2_replica_n2' using configuration from collection authCollection, trusted=true
[junit4] 2> 358435 INFO (qtp1161232537-1885) [n:127.0.0.1:43964_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43964.solr.core.authCollection.shard2.replica_n2' (registry 'solr.core.authCollection.shard2.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1f2b90af
[junit4] 2> 358437 INFO (qtp1161232537-1885) [n:127.0.0.1:43964_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 358437 INFO (qtp1161232537-1885) [n:127.0.0.1:43964_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.c.SolrCore [[authCollection_shard2_replica_n2] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.security.BasicAuthIntegrationTest_FE8EF5D38704C33C-001/tempDir-001/node2/authCollection_shard2_replica_n2], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.security.BasicAuthIntegrationTest_FE8EF5D38704C33C-001/tempDir-001/node2/./authCollection_shard2_replica_n2/data/]
[junit4] 2> 358384 INFO (qtp1307534678-1883) [n:127.0.0.1:41028_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.s.IndexSchema [authCollection_shard3_replica_n4] Schema name=minimal
[junit4] 2> 358469 INFO (qtp1307534678-1883) [n:127.0.0.1:41028_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 358469 INFO (qtp1307534678-1883) [n:127.0.0.1:41028_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.c.CoreContainer Creating SolrCore 'authCollection_shard3_replica_n4' using configuration from collection authCollection, trusted=true
[junit4] 2> 358469 INFO (qtp1307534678-1883) [n:127.0.0.1:41028_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41028.solr.core.authCollection.shard3.replica_n4' (registry 'solr.core.authCollection.shard3.replica_n4') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1f2b90af
[junit4] 2> 358487 INFO (qtp1307534678-1883) [n:127.0.0.1:41028_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 358487 INFO (qtp1307534678-1883) [n:127.0.0.1:41028_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.c.SolrCore [[authCollection_shard3_replica_n4] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.security.BasicAuthIntegrationTest_FE8EF5D38704C33C-001/tempDir-001/node3/authCollection_shard3_replica_n4], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.security.BasicAuthIntegrationTest_FE8EF5D38704C33C-001/tempDir-001/node3/./authCollection_shard3_replica_n4/data/]
[junit4] 2> 359075 INFO (qtp1161232537-1885) [n:127.0.0.1:43964_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 359075 INFO (qtp1161232537-1885) [n:127.0.0.1:43964_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 359085 INFO (qtp1307534678-1883) [n:127.0.0.1:41028_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 359085 INFO (qtp1307534678-1883) [n:127.0.0.1:41028_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 359086 INFO (qtp1307534678-1883) [n:127.0.0.1:41028_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 359086 INFO (qtp1307534678-1883) [n:127.0.0.1:41028_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 359093 INFO (qtp270627030-1904) [n:127.0.0.1:38671_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 359093 INFO (qtp270627030-1904) [n:127.0.0.1:38671_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 359094 INFO (qtp270627030-1904) [n:127.0.0.1:38671_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 359094 INFO (qtp270627030-1904) [n:127.0.0.1:38671_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 359096 INFO (qtp270627030-1904) [n:127.0.0.1:38671_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@75d4f119[authCollection_shard1_replica_n1] main]
[junit4] 2> 359100 INFO (qtp1307534678-1883) [n:127.0.0.1:41028_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.s.SolrIndexSearcher Opening [Searcher@c9c7bb6[authCollection_shard3_replica_n4] main]
[junit4] 2> 359102 INFO (qtp1307534678-1883) [n:127.0.0.1:41028_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 359109 INFO (qtp1161232537-1885) [n:127.0.0.1:43964_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 359109 INFO (qtp1161232537-1885) [n:127.0.0.1:43964_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 359130 INFO (qtp1307534678-1883) [n:127.0.0.1:41028_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 359131 INFO (qtp1307534678-1883) [n:127.0.0.1:41028_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 359131 INFO (qtp270627030-1904) [n:127.0.0.1:38671_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 359131 INFO (qtp1307534678-1883) [n:127.0.0.1:41028_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1595011717874581504
[junit4] 2> 359132 INFO (qtp270627030-1904) [n:127.0.0.1:38671_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 359132 INFO (qtp270627030-1904) [n:127.0.0.1:38671_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 359133 INFO (qtp270627030-1904) [n:127.0.0.1:38671_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1595011717876678656
[junit4] 2> 359145 INFO (qtp1161232537-1885) [n:127.0.0.1:43964_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@10007b26[authCollection_shard2_replica_n2] main]
[junit4] 2> 359178 INFO (searcherExecutor-340-thread-1-processing-n:127.0.0.1:41028_solr x:authCollection_shard3_replica_n4 s:shard3 c:authCollection r:core_node6) [n:127.0.0.1:41028_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.c.SolrCore [authCollection_shard3_replica_n4] Registered new searcher Searcher@c9c7bb6[authCollection_shard3_replica_n4] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 359179 INFO (searcherExecutor-338-thread-1-processing-n:127.0.0.1:38671_solr x:authCollection_shard1_replica_n1 s:shard1 c:authCollection r:core_node3) [n:127.0.0.1:38671_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.c.SolrCore [authCollection_shard1_replica_n1] Registered new searcher Searcher@75d4f119[authCollection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 359195 INFO (qtp1161232537-1885) [n:127.0.0.1:43964_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 359195 INFO (qtp1161232537-1885) [n:127.0.0.1:43964_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 359196 INFO (qtp1161232537-1885) [n:127.0.0.1:43964_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 359196 INFO (qtp1161232537-1885) [n:127.0.0.1:43964_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1595011717942738944
[junit4] 2> 359267 INFO (searcherExecutor-339-thread-1-processing-n:127.0.0.1:43964_solr x:authCollection_shard2_replica_n2 s:shard2 c:authCollection r:core_node5) [n:127.0.0.1:43964_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.c.SolrCore [authCollection_shard2_replica_n2] Registered new searcher Searcher@10007b26[authCollection_shard2_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 359282 INFO (qtp1161232537-1885) [n:127.0.0.1:43964_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.c.ZkShardTerms Successful update terms at /collections/authCollection/terms/shard2 to Terms{values={core_node5=0}, version=0}
[junit4] 2> 359304 INFO (qtp270627030-1904) [n:127.0.0.1:38671_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.c.ZkShardTerms Successful update terms at /collections/authCollection/terms/shard1 to Terms{values={core_node3=0}, version=0}
[junit4] 2> 359318 INFO (qtp1161232537-1885) [n:127.0.0.1:43964_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 359318 INFO (qtp1161232537-1885) [n:127.0.0.1:43964_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 359318 INFO (qtp1161232537-1885) [n:127.0.0.1:43964_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:43964/solr/authCollection_shard2_replica_n2/
[junit4] 2> 359318 INFO (qtp1161232537-1885) [n:127.0.0.1:43964_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 359319 INFO (qtp1161232537-1885) [n:127.0.0.1:43964_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.c.SyncStrategy http://127.0.0.1:43964/solr/authCollection_shard2_replica_n2/ has no replicas
[junit4] 2> 359319 INFO (qtp1161232537-1885) [n:127.0.0.1:43964_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 359340 INFO (qtp1307534678-1883) [n:127.0.0.1:41028_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.c.ZkShardTerms Successful update terms at /collections/authCollection/terms/shard3 to Terms{values={core_node6=0}, version=0}
[junit4] 2> 359353 INFO (qtp1161232537-1885) [n:127.0.0.1:43964_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:43964/solr/authCollection_shard2_replica_n2/ shard2
[junit4] 2> 359369 INFO (zkCallback-192-thread-1-processing-n:127.0.0.1:43964_solr) [n:127.0.0.1:43964_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
[junit4] 2> 359369 INFO (zkCallback-192-thread-3-processing-n:127.0.0.1:43964_solr) [n:127.0.0.1:43964_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
[junit4] 2> 359377 INFO (zkCallback-193-thread-2-processing-n:127.0.0.1:41028_solr) [n:127.0.0.1:41028_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
[junit4] 2> 359377 INFO (zkCallback-196-thread-2-processing-n:127.0.0.1:38671_solr) [n:127.0.0.1:38671_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
[junit4] 2> 359377 INFO (zkCallback-193-thread-1-processing-n:127.0.0.1:41028_solr) [n:127.0.0.1:41028_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
[junit4] 2> 359377 INFO (zkCallback-196-thread-1-processing-n:127.0.0.1:38671_solr) [n:127.0.0.1:38671_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
[junit4] 2> 359405 INFO (qtp1161232537-1885) [n:127.0.0.1:43964_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 359407 INFO (qtp1161232537-1885) [n:127.0.0.1:43964_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node5&collection.configName=conf&newCollection=true&name=authCollection_shard2_replica_n2&action=CREATE&numShards=3&collection=authCollection&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=6643
[junit4] 2> 359498 INFO (qtp270627030-1904) [n:127.0.0.1:38671_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 359499 INFO (qtp270627030-1904) [n:127.0.0.1:38671_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 359499 INFO (qtp270627030-1904) [n:127.0.0.1:38671_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:38671/solr/authCollection_shard1_replica_n1/
[junit4] 2> 359499 INFO (qtp270627030-1904) [n:127.0.0.1:38671_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 359499 INFO (qtp270627030-1904) [n:127.0.0.1:38671_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:38671/solr/authCollection_shard1_replica_n1/ has no replicas
[junit4] 2> 359499 INFO (qtp270627030-1904) [n:127.0.0.1:38671_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 359499 INFO (qtp1307534678-1883) [n:127.0.0.1:41028_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 359499 INFO (qtp1307534678-1883) [n:127.0.0.1:41028_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 359499 INFO (qtp1307534678-1883) [n:127.0.0.1:41028_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:41028/solr/authCollection_shard3_replica_n4/
[junit4] 2> 359499 INFO (qtp1307534678-1883) [n:127.0.0.1:41028_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 359499 INFO (qtp1307534678-1883) [n:127.0.0.1:41028_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.c.SyncStrategy http://127.0.0.1:41028/solr/authCollection_shard3_replica_n4/ has no replicas
[junit4] 2> 359499 INFO (qtp1307534678-1883) [n:127.0.0.1:41028_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 359601 INFO (zkCallback-196-thread-2-processing-n:127.0.0.1:38671_solr) [n:127.0.0.1:38671_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
[junit4] 2> 359601 INFO (zkCallback-196-thread-1-processing-n:127.0.0.1:38671_solr) [n:127.0.0.1:38671_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
[junit4] 2> 359604 INFO (zkCallback-192-thread-3-processing-n:127.0.0.1:43964_solr) [n:127.0.0.1:43964_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
[junit4] 2> 359612 INFO (zkCallback-192-thread-2-processing-n:127.0.0.1:43964_solr) [n:127.0.0.1:43964_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
[junit4] 2> 359612 INFO (zkCallback-193-thread-1-processing-n:127.0.0.1:41028_solr) [n:127.0.0.1:41028_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
[junit4] 2> 359612 INFO (zkCallback-193-thread-2-processing-n:127.0.0.1:41028_solr) [n:127.0.0.1:41028_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
[junit4] 2> 359657 INFO (qtp270627030-1904) [n:127.0.0.1:38671_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:38671/solr/authCollection_shard1_replica_n1/ shard1
[junit4] 2> 359673 INFO (qtp1307534678-1883) [n:127.0.0.1:41028_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:41028/solr/authCollection_shard3_replica_n4/ shard3
[junit4] 2> 359819 INFO (zkCallback-192-thread-2-processing-n:127.0.0.1:43964_solr) [n:127.0.0.1:43964_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
[junit4] 2> 359819 INFO (zkCallback-196-thread-1-processing-n:127.0.0.1:38671_solr) [n:127.0.0.1:38671_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
[junit4] 2> 359819 INFO (zkCallback-193-thread-1-processing-n:127.0.0.1:41028_solr) [n:127.0.0.1:41028_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
[junit4] 2> 359819 INFO (zkCallback-193-thread-2-processing-n:127.0.0.1:41028_solr) [n:127.0.0.1:41028_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
[junit4] 2> 359819 INFO (zkCallback-192-thread-3-processing-n:127.0.0.1:43964_solr) [n:127.0.0.1:43964_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
[junit4] 2> 359820 INFO (zkCallback-196-thread-2-processing-n:127.0.0.1:38671_solr) [n:127.0.0.1:38671_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
[junit4] 2> 359841 INFO (qtp270627030-1904) [n:127.0.0.1:38671_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 359843 INFO (qtp270627030-1904) [n:127.0.0.1:38671_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=authCollection_shard1_replica_n1&action=CREATE&numShards=3&collection=authCollection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=7169
[junit4] 2> 359864 INFO (qtp1307534678-1883) [n:127.0.0.1:41028_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 359866 INFO (qtp1307534678-1883) [n:127.0.0.1:41028_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node6&collection.configName=conf&newCollection=true&name=authCollection_shard3_replica_n4&action=CREATE&numShards=3&collection=authCollection&shard=shard3&wt=javabin&version=2&replicaType=NRT} status=0 QTime=7118
[junit4] 2> 359916 INFO (qtp270627030-1899) [n:127.0.0.1:38671_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 360002 INFO (zkCallback-193-thread-2-processing-n:127.0.0.1:41028_solr) [n:127.0.0.1:41028_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
[junit4] 2> 360002 INFO (zkCallback-193-thread-1-processing-n:127.0.0.1:41028_solr) [n:127.0.0.1:41028_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
[junit4] 2> 360006 INFO (zkCallback-192-thread-3-processing-n:127.0.0.1:43964_solr) [n:127.0.0.1:43964_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
[junit4] 2> 360006 INFO (zkCallback-196-thread-1-processing-n:127.0.0.1:38671_solr) [n:127.0.0.1:38671_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
[junit4] 2> 360006 INFO (zkCallback-192-thread-2-processing-n:127.0.0.1:43964_solr) [n:127.0.0.1:43964_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.j
[...truncated too long message...]
04792 INFO (jetty-closer-162-thread-3) [ ] o.e.j.s.session Stopped scavenging
[junit4] 2> 304793 INFO (jetty-closer-162-thread-2) [ ] o.a.s.c.Overseer Overseer (id=72284559266086939-127.0.0.1:34002_solr-n_0000000005) closing
[junit4] 2> 304793 INFO (OverseerStateUpdate-72284559266086939-127.0.0.1:34002_solr-n_0000000005) [n:127.0.0.1:34002_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:34002_solr
[junit4] 2> 304937 WARN (OverseerAutoScalingTriggerThread-72284559266086939-127.0.0.1:34002_solr-n_0000000005) [n:127.0.0.1:34002_solr ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 304975 INFO (jetty-closer-162-thread-4) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@2f07d4e2{/solr,null,UNAVAILABLE}
[junit4] 2> 304976 INFO (jetty-closer-162-thread-4) [ ] o.e.j.s.session Stopped scavenging
[junit4] 2> 304993 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x100ce6c75cd001d, likely client has closed socket
[junit4] 2> 305030 INFO (jetty-closer-162-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@5a106866{/solr,null,UNAVAILABLE}
[junit4] 2> 305030 INFO (jetty-closer-162-thread-1) [ ] o.e.j.s.session Stopped scavenging
[junit4] 2> 305099 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x100ce6c75cd001b, likely client has closed socket
[junit4] 2> 305105 INFO (jetty-closer-162-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@d2a2511{/solr,null,UNAVAILABLE}
[junit4] 2> 305105 INFO (jetty-closer-162-thread-2) [ ] o.e.j.s.session Stopped scavenging
[junit4] 2> 305106 ERROR (SUITE-TestCollectionStateWatchers-seed#[811295EA81EB6161]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 305140 INFO (SUITE-TestCollectionStateWatchers-seed#[811295EA81EB6161]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:42558 42558
[junit4] 2> 305214 INFO (Thread-120) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:42558 42558
[junit4] 2> 305214 WARN (Thread-120) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 22 /solr/aliases.json
[junit4] 2> 10 /solr/security.json
[junit4] 2> 8 /solr/configs/config
[junit4] 2> 6 /solr/collections/falsepredicate/collectionprops.json
[junit4] 2> 4 /solr/collections/testcollection/collectionprops.json
[junit4] 2> 4 /solr/collections/stateformat1/state.json
[junit4] 2> 3 /solr/collections/stateformat1/terms/shard1
[junit4] 2> 3 /solr/collections/waitforstate/terms/shard1
[junit4] 2> 3 /solr/collections/falsepredicate/terms/shard4
[junit4] 2> 3 /solr/collections/falsepredicate/terms/shard3
[junit4] 2> 3 /solr/collections/falsepredicate/terms/shard2
[junit4] 2> 3 /solr/collections/falsepredicate/terms/shard1
[junit4] 2> 3 /solr/collections/tobedeleted/terms/shard1
[junit4] 2> 3 /solr/collections/currentstate/terms/shard1
[junit4] 2> 2 /solr/collections/currentstate/collectionprops.json
[junit4] 2> 2 /solr/collections/testcollection/terms/shard1
[junit4] 2> 2 /solr/collections/testcollection/terms/shard2
[junit4] 2> 2 /solr/collections/testcollection/terms/shard3
[junit4] 2> 2 /solr/collections/testcollection/terms/shard4
[junit4] 2> 2 /solr/collections/delayed/terms/shard1
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 49 /solr/collections/testcollection/state.json
[junit4] 2> 45 /solr/collections/falsepredicate/state.json
[junit4] 2> 22 /solr/clusterprops.json
[junit4] 2> 22 /solr/clusterstate.json
[junit4] 2> 17 /solr/collections/waitforstate/state.json
[junit4] 2> 12 /solr/collections/currentstate/state.json
[junit4] 2> 7 /solr/collections/delayed/state.json
[junit4] 2> 6 /solr/collections/tobedeleted/state.json
[junit4] 2> 5 /solr/collections/stateformat1/state.json
[junit4] 2> 3 /solr/overseer_elect/election/72284559266086940-127.0.0.1:39238_solr-n_0000000006
[junit4] 2> 2 /solr/overseer_elect/election/72284559266086924-127.0.0.1:39238_solr-n_0000000000
[junit4] 2> 2 /solr/autoscaling.json
[junit4] 2> 2 /solr/overseer_elect/election/72284559266086922-127.0.0.1:34963_solr-n_0000000001
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 22 /solr/live_nodes
[junit4] 2> 22 /solr/collections
[junit4] 2> 2 /solr/autoscaling/events/.scheduled_maintenance
[junit4] 2> 2 /solr/overseer/queue
[junit4] 2> 2 /solr/autoscaling/events/.auto_add_replicas
[junit4] 2> 2 /solr/overseer/collection-queue-work
[junit4] 2> 2 /solr/overseer/queue-work
[junit4] 2>
[junit4] 2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-solrj/test/J2/temp/solr.common.cloud.TestCollectionStateWatchers_811295EA81EB6161-001
[junit4] 2> NOTE: test params are: codec=CheapBastard, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@3e9d3604), locale=el, timezone=GMT
[junit4] 2> NOTE: Linux 4.4.0-112-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=1,free=228901328,total=491257856
[junit4] 2> NOTE: All tests run in this JVM: [TestJavaBinCodec, ModifiableSolrParamsTest, TestCloudSolrClientConnections, NormalizeEvaluatorTest, CoalesceEvaluatorTest, CommonParamsTest, SolrExampleXMLTest, LessThanEqualToEvaluatorTest, TestCollectionAdminRequest, CosineEvaluatorTest, SolrExampleBinaryTest, DocumentAnalysisResponseTest, HyperbolicSineEvaluatorTest, TangentEvaluatorTest, NoOpResponseParserTest, RegressionEvaluatorTest, AddEvaluatorTest, JdbcDriverTest, LargeVolumeJettyTest, CloudSolrClientTest, PowerEvaluatorTest, TestCoreAdmin, SolrExceptionTest, TestCollectionStateWatchers]
[junit4] Completed [77/147 (1!)] on J2 in 114.67s, 9 tests, 1 failure <<< FAILURES!
[...truncated 2346 lines...]
[junit4] Suite: org.apache.solr.analytics.facet.QueryFacetTest
[junit4] 2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/contrib/solr-analytics/test/J1/temp/solr.analytics.facet.QueryFacetTest_6005039964E31AA5-001/init-core-data-001
[junit4] 2> Mar 15, 2018 3:05:16 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=936, maxMBSortInHeap=6.9160233339428725, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@51194c70), locale=id, timezone=America/Belem
[junit4] 2> NOTE: Linux 4.4.0-112-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=1,free=274118312,total=312475648
[junit4] 2> NOTE: All tests run in this JVM: [ExistsFunctionTest, LegacyRangeFacetTest, LegacyFieldFacetExtrasCloudTest, LegacyAbstractAnalyticsFacetTest, CastingDoubleValueStreamTest, FloorFunctionTest, CastingLongValueStreamTest, CastingStringValueTest, LegacyAbstractAnalyticsTest, QueryFacetTest]
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=QueryFacetTest -Dtests.seed=6005039964E31AA5 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=id -Dtests.timezone=America/Belem -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] ERROR 0.00s J1 | QueryFacetTest (suite) <<<
[junit4] > Throwable #1: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:33906/solr: Expected mime type application/octet-stream but got text/html. <h1>Bad Message 400</h1><pre>reason: Illegal character 0x0</pre>
[junit4] > at __randomizedtesting.SeedInfo.seed([6005039964E31AA5]:0)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:607)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
[junit4] > at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
[junit4] > at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
[junit4] > at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1105)
[junit4] > at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:885)
[junit4] > at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:818)
[junit4] > at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
[junit4] > at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
[junit4] > at org.apache.solr.analytics.SolrAnalyticsTestCase.setupCollection(SolrAnalyticsTestCase.java:62)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] Completed [28/78 (1!)] on J1 in 4.59s, 0 tests, 1 error <<< FAILURES!
[...truncated 46411 lines...]
[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1502 - Failure
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1502/
1 tests failed.
FAILED: org.apache.lucene.index.TestIndexSorting.testRandom3
Error Message:
Java heap space
Stack Trace:
java.lang.OutOfMemoryError: Java heap space
at __randomizedtesting.SeedInfo.seed([EBA222EFD1964B12:497A6C35B5646214]:0)
at org.apache.lucene.util.packed.Packed64.<init>(Packed64.java:73)
at org.apache.lucene.util.packed.PackedInts.getMutable(PackedInts.java:972)
at org.apache.lucene.util.packed.PackedInts.getMutable(PackedInts.java:939)
at org.apache.lucene.util.packed.GrowableWriter.ensureCapacity(GrowableWriter.java:80)
at org.apache.lucene.util.packed.GrowableWriter.set(GrowableWriter.java:88)
at org.apache.lucene.util.packed.AbstractPagedMutable.set(AbstractPagedMutable.java:98)
at org.apache.lucene.util.fst.NodeHash.addNew(NodeHash.java:152)
at org.apache.lucene.util.fst.NodeHash.rehash(NodeHash.java:169)
at org.apache.lucene.util.fst.NodeHash.add(NodeHash.java:133)
at org.apache.lucene.util.fst.Builder.compileNode(Builder.java:200)
at org.apache.lucene.util.fst.Builder.freezeTail(Builder.java:296)
at org.apache.lucene.util.fst.Builder.add(Builder.java:400)
at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.writeFST(MemoryDocValuesConsumer.java:373)
at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.addSortedField(MemoryDocValuesConsumer.java:416)
at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.addSortedField(MemoryDocValuesConsumer.java:406)
at org.apache.lucene.codecs.DocValuesConsumer.mergeSortedField(DocValuesConsumer.java:527)
at org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:139)
at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:151)
at org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:194)
at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:149)
at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4441)
at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4081)
at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2245)
at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5096)
at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1730)
at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1462)
at org.apache.lucene.index.TestIndexSorting.testRandom3(TestIndexSorting.java:2294)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
Build Log:
[...truncated 1556 lines...]
[junit4] Suite: org.apache.lucene.index.TestIndexSorting
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestIndexSorting -Dtests.method=testRandom3 -Dtests.seed=EBA222EFD1964B12 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=ga-IE -Dtests.timezone=SystemV/EST5 -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
[junit4] ERROR 298s J0 | TestIndexSorting.testRandom3 <<<
[junit4] > Throwable #1: java.lang.OutOfMemoryError: Java heap space
[junit4] > at __randomizedtesting.SeedInfo.seed([EBA222EFD1964B12:497A6C35B5646214]:0)
[junit4] > at org.apache.lucene.util.packed.Packed64.<init>(Packed64.java:73)
[junit4] > at org.apache.lucene.util.packed.PackedInts.getMutable(PackedInts.java:972)
[junit4] > at org.apache.lucene.util.packed.PackedInts.getMutable(PackedInts.java:939)
[junit4] > at org.apache.lucene.util.packed.GrowableWriter.ensureCapacity(GrowableWriter.java:80)
[junit4] > at org.apache.lucene.util.packed.GrowableWriter.set(GrowableWriter.java:88)
[junit4] > at org.apache.lucene.util.packed.AbstractPagedMutable.set(AbstractPagedMutable.java:98)
[junit4] > at org.apache.lucene.util.fst.NodeHash.addNew(NodeHash.java:152)
[junit4] > at org.apache.lucene.util.fst.NodeHash.rehash(NodeHash.java:169)
[junit4] > at org.apache.lucene.util.fst.NodeHash.add(NodeHash.java:133)
[junit4] > at org.apache.lucene.util.fst.Builder.compileNode(Builder.java:200)
[junit4] > at org.apache.lucene.util.fst.Builder.freezeTail(Builder.java:296)
[junit4] > at org.apache.lucene.util.fst.Builder.add(Builder.java:400)
[junit4] > at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.writeFST(MemoryDocValuesConsumer.java:373)
[junit4] > at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.addSortedField(MemoryDocValuesConsumer.java:416)
[junit4] > at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.addSortedField(MemoryDocValuesConsumer.java:406)
[junit4] > at org.apache.lucene.codecs.DocValuesConsumer.mergeSortedField(DocValuesConsumer.java:527)
[junit4] > at org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:139)
[junit4] > at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:151)
[junit4] > at org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:194)
[junit4] > at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:149)
[junit4] > at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4441)
[junit4] > at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4081)
[junit4] > at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
[junit4] > at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2245)
[junit4] > at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5096)
[junit4] > at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1730)
[junit4] > at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1462)
[junit4] > at org.apache.lucene.index.TestIndexSorting.testRandom3(TestIndexSorting.java:2294)
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.index.TestIndexSorting_EBA222EFD1964B12-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {docs=PostingsFormat(name=Memory), sparse_text=BlockTreeOrds(blocksize=128), norms=PostingsFormat(name=Memory), positions=FST50, id=PostingsFormat(name=Memory), term_vectors=PostingsFormat(name=Memory)}, docValues:{multi_valued_long=DocValuesFormat(name=Lucene70), foo=DocValuesFormat(name=Lucene70), numeric=DocValuesFormat(name=Memory), multi_valued_numeric=DocValuesFormat(name=Memory), float=DocValuesFormat(name=Asserting), long=DocValuesFormat(name=Asserting), points=DocValuesFormat(name=Lucene70), sparse_int=DocValuesFormat(name=Asserting), bar=DocValuesFormat(name=Memory), multi_valued_double=DocValuesFormat(name=Memory), docs=DocValuesFormat(name=Memory), id=DocValuesFormat(name=Memory), multi_valued_int=DocValuesFormat(name=Lucene70), multi_valued_bytes=DocValuesFormat(name=Memory), sparse_text=DocValuesFormat(name=Asserting), double=DocValuesFormat(name=Memory), positions=DocValuesFormat(name=Lucene70), sparse_binary=DocValuesFormat(name=Asserting), int=DocValuesFormat(name=Memory), sorted=DocValuesFormat(name=Memory), multi_valued_string=DocValuesFormat(name=Memory), norms=DocValuesFormat(name=Memory), sparse=DocValuesFormat(name=Asserting), bytes=DocValuesFormat(name=Memory), binary=DocValuesFormat(name=Memory), dense_int=DocValuesFormat(name=Memory), multi_valued_float=DocValuesFormat(name=Lucene70), term_vectors=DocValuesFormat(name=Memory)}, maxPointsInLeafNode=284, maxMBSortInHeap=6.5621828219182134, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@46b2e3da), locale=ga-IE, timezone=SystemV/EST5
[junit4] 2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=319917056,total=414187520
[junit4] 2> NOTE: All tests run in this JVM: [TestUnicodeUtil, TestLucene50StoredFieldsFormatHighCompression, TestVirtualMethod, TestIndexInput, TestFieldInvertState, TestIntBlockPool, TestVersion, TestMultiPhraseEnum, TestFlushByRamOrCountsPolicy, TestDocInverterPerFieldErrorInfo, TestAllFilesDetectTruncation, TestCachingCollector, TestDoubleRangeFieldQueries, TestCharArrayMap, TestRegExp, TestFilterIterator, TestPostingsOffsets, TestIntroSorter, TestSimpleSearchEquivalence, TestIntsRef, TestNormsFieldExistsQuery, TestSimpleFSLockFactory, TestFilterLeafReader, TestAttributeSource, TestMultiTermsEnum, TestLRUQueryCache, TestFieldType, TestNeverDelete, TestDocumentWriter, TestIndependenceSaturated, TestIndexWriterNRTIsCurrent, TestRollingBuffer, TestSortedSetSelector, TestTrackingDirectoryWrapper, Test2BNumericDocValues, TestDocValuesIndexing, TestSort, TestSPIClassIterator, TestIndexWriterExceptions, Test2BSortedDocValuesFixedSorted, TestLucene70NormsFormat, TestDocIdsWriter, TestBasicModelG, TestFieldMaskingSpanQuery, TestBasicModelIne, TestReadOnlyIndex, TestOneMergeWrappingMergePolicy, TestBinaryTerms, TestWANDScorer, TestReqExclBulkScorer, TestStressIndexing, LimitedFiniteStringsIteratorTest, TestInPlaceMergeSorter, TestBagOfPositions, TestForceMergeForever, TestSearchAfter, TestDeterminizeLexicon, TestDeletionPolicy, Test2BPositions, TestTermsEnum, TestSloppyPhraseQuery, TestIndexWriterForceMerge, TestMultiPhraseQuery, TestWordlistLoader, TestPersistentSnapshotDeletionPolicy, TestParallelTermEnum, TestPagedBytes, TestTragicIndexWriterDeadlock, TestMultiThreadTermVectors, TestLiveFieldValues, TestFilterDirectoryReader, TestDocIdSetBuilder, TestDuelingCodecsAtNight, TestLMJelinekMercerSimilarity, TestDocsAndPositions, TestTopFieldCollector, TestControlledRealTimeReopenThread, TestIndexWriterDeleteByQuery, TestNearSpansOrdered, TestFlex, TestDuelingCodecs, TestBoolean2, TestNorms, TestStandardAnalyzer, TestDirectoryReaderReopen, TestTopDocsMerge, TestSnapshotDeletionPolicy, TestIndexWriterUnicode, TestPerFieldPostingsFormat, TestBufferedIndexInput, TestConsistentFieldNumbers, TestWeakIdentityMap, TestDirectoryReader, TestOmitNorms, TestFastDecompressionMode, TestTransactions, TestPerFieldPostingsFormat2, Test2BDocs, TestSpans, TestCustomNorms, TestSpanExplanations, TestFastCompressionMode, TestRAMDirectory, TestLazyProxSkipping, TestMergeSchedulerExternal, TestCodecs, TestParallelLeafReader, TestMultiFields, TestFuzzyQuery, TestPayloads, TestSimilarity2, TestNoDeletionPolicy, TestSimpleExplanationsOfNonMatches, TestPerSegmentDeletes, TestPrefixInBooleanQuery, TestUniqueTermCount, TestComplexExplanationsOfNonMatches, TestSortRandom, TestDocument, TestTermScorer, TestDirectory, TestMultiLevelSkipList, TestSpanMultiTermQueryWrapper, TestBooleanQueryVisitSubscorers, TestSearchForDuplicates, TestElevationComparator, TestRecyclingIntBlockAllocator, TestIndexWriterConfig, TestConjunctions, TestMultiTermQueryRewrites, TestPositionIncrement, TestDateSort, TestSimilarity, TestSmallFloat, TestSearch, TestCachingTokenFilter, TestDateTools, TestCharsRef, TestPriorityQueue, TestBinaryDocument, TestTotalHitCountCollector, TestRamUsageEstimator, TestSpanFirstQuery, TestIsCurrent, TestNoMergeScheduler, TestNamedSPILoader, TestPhrasePrefixQuery, TestByteBlockPool, TestBytesRef, TestDemo, TestCharFilter, TestRollback, TestFieldInfos, TestTwoPhaseCommitTool, TestCloseableThreadLocal, TestCodecHoldsOpenFiles, MultiCollectorTest, TestNGramPhraseQuery, TestIndexCommit, Test2BPostingsBytes, TestPositiveScoresOnlyCollector, TestIOUtils, TestNewestSegment, Test2BBinaryDocValues, Test2BPagedBytes, TestCharArraySet, TestCharacterUtils, TestDelegatingAnalyzerWrapper, TestStopFilter, TestPackedTokenAttributeImpl, TestCodecUtil, TestCompetitiveFreqNormAccumulator, TestBlockPostingsFormat, TestBlockPostingsFormat2, TestForUtil, TestLucene50StoredFieldsFormat, TestLucene60PointsFormat, TestIndexedDISI, TestLucene70SegmentInfoFormat, TestLongRange, TestGeoUtils, TestPolygon, Test2BPoints, TestAllFilesCheckIndexHeader, TestCustomTermFreq, TestDocValues, TestIndexReaderClose, TestIndexSorting]
[junit4] Completed [381/486 (1!)] on J0 in 321.24s, 50 tests, 1 error <<< FAILURES!
[...truncated 345 lines...]
[junit4] JVM J0: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp/junit4-J0-20180313_043605_9954030332040197151159.sysout
[junit4] >>> JVM J0 emitted unexpected output (verbatim) ----
[junit4] java.lang.OutOfMemoryError: Java heap space
[junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps/java_pid28125.hprof ...
[junit4] Heap dump file created [493357855 bytes in 1.971 secs]
[junit4] <<< JVM J0: EOF ----
[...truncated 22825 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:636: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:570: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid28125.hprof
Total time: 329 minutes 42 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1501 - Still
Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1501/
4 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsUnloadDistributedZkTest
Error Message:
71 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsUnloadDistributedZkTest: 1) Thread[id=6823, name=qtp725343090-6823, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140) at org.eclipse.jetty.io.ManagedSelector$$Lambda$240/1428089623.run(Unknown Source) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=6912, name=OverseerAutoScalingTriggerThread-72266723021619210-127.0.0.1:33434_wm_-n_0000000001, state=WAITING, group=Overseer autoscaling triggers] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:177) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=6881, name=Scheduler-691069682, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=6934, name=AutoscalingActionExecutor-955-thread-1-processing-n:127.0.0.1:33434_wm_, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70) at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340) at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176) at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78) at org.apache.solr.cloud.autoscaling.ComputePlanAction.process(ComputePlanAction.java:72) at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:307) at org.apache.solr.cloud.autoscaling.ScheduledTriggers$$Lambda$491/1809276628.run(Unknown Source) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$9/751561345.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=6874, name=qtp836920967-6874, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140) at org.eclipse.jetty.io.ManagedSelector$$Lambda$240/1428089623.run(Unknown Source) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=6802, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=6805, name=zkConnectionManagerCallback-489-thread-1, state=WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 8) Thread[id=6787, name=Scheduler-2087371397, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 9) Thread[id=6822, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 10) Thread[id=6779, name=qtp1025705465-6779, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 11) Thread[id=6824, name=qtp725343090-6824, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663) at java.lang.Thread.run(Thread.java:748) 12) Thread[id=6897, name=TEST-HdfsUnloadDistributedZkTest.test-seed#[28E698689983B5BE]-SendThread(127.0.0.1:36774), state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063) 13) Thread[id=6899, name=zkConnectionManagerCallback-515-thread-1, state=WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 14) Thread[id=6841, name=TEST-HdfsUnloadDistributedZkTest.test-seed#[28E698689983B5BE]-EventThread, state=WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502) 15) Thread[id=6791, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 16) Thread[id=6890, name=TEST-HdfsUnloadDistributedZkTest.test-seed#[28E698689983B5BE]-SendThread(127.0.0.1:36774), state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1054) 17) Thread[id=6806, name=Scheduler-372419654, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 18) Thread[id=6914, name=ScheduledTrigger-954-thread-2, state=WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 19) Thread[id=6910, name=OverseerStateUpdate-72266723021619210-127.0.0.1:33434_wm_-n_0000000001, state=TIMED_WAITING, group=Overseer state updater.] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70) at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340) at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:415) at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:145) at java.lang.Thread.run(Thread.java:748) 20) Thread[id=6785, name=qtp1025705465-6785, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 21) Thread[id=6916, name=ScheduledTrigger-954-thread-4, state=WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 22) Thread[id=6876, name=qtp836920967-6876, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663) at java.lang.Thread.run(Thread.java:748) 23) Thread[id=6798, name=zkConnectionManagerCallback-484-thread-1, state=WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 24) Thread[id=6826, name=qtp725343090-6826-acceptor-0@6d47a35e-ServerConnector@68daf8b6{HTTP/1.1,[http/1.1]}{127.0.0.1:42068}, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:379) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:638) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 25) Thread[id=6835, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 26) Thread[id=6896, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 27) Thread[id=6804, name=TEST-HdfsUnloadDistributedZkTest.test-seed#[28E698689983B5BE]-EventThread, state=WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502) 28) Thread[id=6861, name=searcherExecutor-940-thread-1, state=WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 29) Thread[id=6781, name=qtp1025705465-6781-acceptor-0@7eb8bc38-ServerConnector@707f4d9f{HTTP/1.1,[http/1.1]}{127.0.0.1:33434}, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:379) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:638) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 30) Thread[id=6911, name=OverseerCollectionConfigSetProcessor-72266723021619210-127.0.0.1:33434_wm_-n_0000000001, state=TIMED_WAITING, group=Overseer collection creation process.] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70) at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340) at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:392) at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:193) at java.lang.Thread.run(Thread.java:748) 31) Thread[id=6831, name=Scheduler-713182388, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 32) Thread[id=6792, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 33) Thread[id=6875, name=qtp836920967-6875-acceptor-0@370362a5-ServerConnector@bb69cec{HTTP/1.1,[http/1.1]}{127.0.0.1:45474}, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:379) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:638) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 34) Thread[id=6829, name=qtp725343090-6829, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663) at java.lang.Thread.run(Thread.java:748) 35) Thread[id=6892, name=zkConnectionManagerCallback-510-thread-1, state=WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 36) Thread[id=6879, name=qtp836920967-6879, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663) at java.lang.Thread.run(Thread.java:748) 37) Thread[id=6780, name=qtp1025705465-6780, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663) at java.lang.Thread.run(Thread.java:748) 38) Thread[id=6825, name=qtp725343090-6825, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663) at java.lang.Thread.run(Thread.java:748) 39) Thread[id=6880, name=qtp836920967-6880, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663) at java.lang.Thread.run(Thread.java:748) 40) Thread[id=6913, name=ScheduledTrigger-954-thread-1, state=WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 41) Thread[id=6842, name=zkConnectionManagerCallback-497-thread-1, state=WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 42) Thread[id=6844, name=Thread-4928, state=WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1731) 43) Thread[id=6885, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 44) Thread[id=6786, name=qtp1025705465-6786, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663) at java.lang.Thread.run(Thread.java:748) 45) Thread[id=6877, name=qtp836920967-6877, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663) at java.lang.Thread.run(Thread.java:748) 46) Thread[id=6784, name=qtp1025705465-6784, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663) at java.lang.Thread.run(Thread.java:748) 47) Thread[id=6886, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 48) Thread[id=6851, name=Scheduler-851849217, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 49) Thread[id=6828, name=qtp725343090-6828, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663) at java.lang.Thread.run(Thread.java:748) 50) Thread[id=6782, name=qtp1025705465-6782, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 51) Thread[id=6873, name=qtp836920967-6873, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140) at org.eclipse.jetty.io.ManagedSelector$$Lambda$240/1428089623.run(Unknown Source) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 52) Thread[id=6800, name=Thread-4912, state=WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1731) 53) Thread[id=6830, name=qtp725343090-6830, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663) at java.lang.Thread.run(Thread.java:748) 54) Thread[id=6827, name=qtp725343090-6827, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 55) Thread[id=6783, name=qtp1025705465-6783, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663) at java.lang.Thread.run(Thread.java:748) 56) Thread[id=6847, name=TEST-HdfsUnloadDistributedZkTest.test-seed#[28E698689983B5BE]-SendThread(127.0.0.1:36774), state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063) 57) Thread[id=6803, name=TEST-HdfsUnloadDistributedZkTest.test-seed#[28E698689983B5BE]-SendThread(127.0.0.1:36774), state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063) 58) Thread[id=6849, name=zkConnectionManagerCallback-502-thread-1, state=WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 59) Thread[id=6840, name=TEST-HdfsUnloadDistributedZkTest.test-seed#[28E698689983B5BE]-SendThread(127.0.0.1:36774), state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063) 60) Thread[id=6898, name=TEST-HdfsUnloadDistributedZkTest.test-seed#[28E698689983B5BE]-EventThread, state=WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502) 61) Thread[id=6797, name=TEST-HdfsUnloadDistributedZkTest.test-seed#[28E698689983B5BE]-EventThread, state=WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502) 62) Thread[id=6796, name=TEST-HdfsUnloadDistributedZkTest.test-seed#[28E698689983B5BE]-SendThread(127.0.0.1:36774), state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:230) at org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1249) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1173) 63) Thread[id=6817, name=searcherExecutor-929-thread-1, state=WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 64) Thread[id=6894, name=Thread-4944, state=WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1731) 65) Thread[id=6891, name=TEST-HdfsUnloadDistributedZkTest.test-seed#[28E698689983B5BE]-EventThread, state=WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502) 66) Thread[id=6878, name=qtp836920967-6878, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663) at java.lang.Thread.run(Thread.java:748) 67) Thread[id=6915, name=ScheduledTrigger-954-thread-3, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:316) at org.apache.solr.client.solrj.impl.ZkDistribStateManager.hasData(ZkDistribStateManager.java:58) at org.apache.solr.cloud.autoscaling.TriggerBase.saveState(TriggerBase.java:203) at org.apache.solr.cloud.autoscaling.ScheduledTriggers$TriggerWrapper.run(ScheduledTriggers.java:582) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 68) Thread[id=6846, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 69) Thread[id=6836, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 70) Thread[id=6848, name=TEST-HdfsUnloadDistributedZkTest.test-seed#[28E698689983B5BE]-EventThread, state=WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502) 71) Thread[id=6864, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 71 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsUnloadDistributedZkTest:
1) Thread[id=6823, name=qtp725343090-6823, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
at org.eclipse.jetty.io.ManagedSelector$$Lambda$240/1428089623.run(Unknown Source)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=6912, name=OverseerAutoScalingTriggerThread-72266723021619210-127.0.0.1:33434_wm_-n_0000000001, state=WAITING, group=Overseer autoscaling triggers]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:177)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=6881, name=Scheduler-691069682, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=6934, name=AutoscalingActionExecutor-955-thread-1-processing-n:127.0.0.1:33434_wm_, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
at org.apache.solr.cloud.autoscaling.ComputePlanAction.process(ComputePlanAction.java:72)
at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:307)
at org.apache.solr.cloud.autoscaling.ScheduledTriggers$$Lambda$491/1809276628.run(Unknown Source)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$9/751561345.run(Unknown Source)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=6874, name=qtp836920967-6874, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
at org.eclipse.jetty.io.ManagedSelector$$Lambda$240/1428089623.run(Unknown Source)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=6802, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=6805, name=zkConnectionManagerCallback-489-thread-1, state=WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
8) Thread[id=6787, name=Scheduler-2087371397, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
9) Thread[id=6822, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
10) Thread[id=6779, name=qtp1025705465-6779, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
11) Thread[id=6824, name=qtp725343090-6824, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
at java.lang.Thread.run(Thread.java:748)
12) Thread[id=6897, name=TEST-HdfsUnloadDistributedZkTest.test-seed#[28E698689983B5BE]-SendThread(127.0.0.1:36774), state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)
at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
13) Thread[id=6899, name=zkConnectionManagerCallback-515-thread-1, state=WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
14) Thread[id=6841, name=TEST-HdfsUnloadDistributedZkTest.test-seed#[28E698689983B5BE]-EventThread, state=WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)
15) Thread[id=6791, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
16) Thread[id=6890, name=TEST-HdfsUnloadDistributedZkTest.test-seed#[28E698689983B5BE]-SendThread(127.0.0.1:36774), state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1054)
17) Thread[id=6806, name=Scheduler-372419654, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
18) Thread[id=6914, name=ScheduledTrigger-954-thread-2, state=WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
19) Thread[id=6910, name=OverseerStateUpdate-72266723021619210-127.0.0.1:33434_wm_-n_0000000001, state=TIMED_WAITING, group=Overseer state updater.]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:415)
at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:145)
at java.lang.Thread.run(Thread.java:748)
20) Thread[id=6785, name=qtp1025705465-6785, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
21) Thread[id=6916, name=ScheduledTrigger-954-thread-4, state=WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
22) Thread[id=6876, name=qtp836920967-6876, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
at java.lang.Thread.run(Thread.java:748)
23) Thread[id=6798, name=zkConnectionManagerCallback-484-thread-1, state=WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
24) Thread[id=6826, name=qtp725343090-6826-acceptor-0@6d47a35e-ServerConnector@68daf8b6{HTTP/1.1,[http/1.1]}{127.0.0.1:42068}, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:379)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:638)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
25) Thread[id=6835, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
26) Thread[id=6896, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
27) Thread[id=6804, name=TEST-HdfsUnloadDistributedZkTest.test-seed#[28E698689983B5BE]-EventThread, state=WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)
28) Thread[id=6861, name=searcherExecutor-940-thread-1, state=WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
29) Thread[id=6781, name=qtp1025705465-6781-acceptor-0@7eb8bc38-ServerConnector@707f4d9f{HTTP/1.1,[http/1.1]}{127.0.0.1:33434}, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:379)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:638)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
30) Thread[id=6911, name=OverseerCollectionConfigSetProcessor-72266723021619210-127.0.0.1:33434_wm_-n_0000000001, state=TIMED_WAITING, group=Overseer collection creation process.]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:392)
at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:193)
at java.lang.Thread.run(Thread.java:748)
31) Thread[id=6831, name=Scheduler-713182388, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
32) Thread[id=6792, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
33) Thread[id=6875, name=qtp836920967-6875-acceptor-0@370362a5-ServerConnector@bb69cec{HTTP/1.1,[http/1.1]}{127.0.0.1:45474}, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:379)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:638)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
34) Thread[id=6829, name=qtp725343090-6829, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
at java.lang.Thread.run(Thread.java:748)
35) Thread[id=6892, name=zkConnectionManagerCallback-510-thread-1, state=WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
36) Thread[id=6879, name=qtp836920967-6879, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
at java.lang.Thread.run(Thread.java:748)
37) Thread[id=6780, name=qtp1025705465-6780, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
at java.lang.Thread.run(Thread.java:748)
38) Thread[id=6825, name=qtp725343090-6825, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
at java.lang.Thread.run(Thread.java:748)
39) Thread[id=6880, name=qtp836920967-6880, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
at java.lang.Thread.run(Thread.java:748)
40) Thread[id=6913, name=ScheduledTrigger-954-thread-1, state=WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
41) Thread[id=6842, name=zkConnectionManagerCallback-497-thread-1, state=WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
42) Thread[id=6844, name=Thread-4928, state=WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at org.apache.solr.core.CloserThread.run(CoreContainer.java:1731)
43) Thread[id=6885, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
44) Thread[id=6786, name=qtp1025705465-6786, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
at java.lang.Thread.run(Thread.java:748)
45) Thread[id=6877, name=qtp836920967-6877, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
at java.lang.Thread.run(Thread.java:748)
46) Thread[id=6784, name=qtp1025705465-6784, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
at java.lang.Thread.run(Thread.java:748)
47) Thread[id=6886, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
48) Thread[id=6851, name=Scheduler-851849217, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
49) Thread[id=6828, name=qtp725343090-6828, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
at java.lang.Thread.run(Thread.java:748)
50) Thread[id=6782, name=qtp1025705465-6782, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
51) Thread[id=6873, name=qtp836920967-6873, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouK
[...truncated too long message...]
ookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkStateReader.getAutoScalingConfig(ZkStateReader.java:196)
[junit4] 2> at org.apache.solr.common.cloud.ZkStateReader.getAutoScalingConfig(ZkStateReader.java:182)
[junit4] 2> at org.apache.solr.cloud.ZkController.createEphemeralLiveNode(ZkController.java:974)
[junit4] 2> at org.apache.solr.cloud.ZkController.access$500(ZkController.java:138)
[junit4] 2> at org.apache.solr.cloud.ZkController$1.command(ZkController.java:363)
[junit4] 2> at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:167)
[junit4] 2> at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57)
[junit4] 2> at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:141)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$1.lambda$process$0(SolrZkClient.java:267)
[junit4] 2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
[junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2>
[junit4] 2> 2044005 ERROR (zkConnectionManagerCallback-1772-thread-1-processing-n:127.0.0.1:34281_solr) [n:127.0.0.1:34281_solr ] o.a.s.c.c.DefaultConnectionStrategy Reconnect to ZooKeeper failed:org.apache.solr.common.cloud.ZooKeeperException:
[junit4] 2> at org.apache.solr.cloud.ZkController$1.command(ZkController.java:415)
[junit4] 2> at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:167)
[junit4] 2> at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57)
[junit4] 2> at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:141)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$1.lambda$process$0(SolrZkClient.java:267)
[junit4] 2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
[junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkStateReader.getAutoScalingConfig(ZkStateReader.java:196)
[junit4] 2> at org.apache.solr.common.cloud.ZkStateReader.getAutoScalingConfig(ZkStateReader.java:182)
[junit4] 2> at org.apache.solr.cloud.ZkController.createEphemeralLiveNode(ZkController.java:974)
[junit4] 2> at org.apache.solr.cloud.ZkController.access$500(ZkController.java:138)
[junit4] 2> at org.apache.solr.cloud.ZkController$1.command(ZkController.java:363)
[junit4] 2> ... 10 more
[junit4] 2>
[junit4] 2> 2044005 WARN (zkConnectionManagerCallback-1772-thread-1-processing-n:127.0.0.1:34281_solr) [n:127.0.0.1:34281_solr ] o.a.s.c.c.DefaultConnectionStrategy Reconnect to ZooKeeper failed
[junit4] 2> 2044005 INFO (zkConnectionManagerCallback-1772-thread-1-processing-n:127.0.0.1:34281_solr) [n:127.0.0.1:34281_solr ] o.a.s.c.c.ConnectionManager zkClient Connected:true
[junit4] 2> 2044006 INFO (jetty-closer-1662-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@7750aab1{/solr,null,UNAVAILABLE}
[junit4] 2> 2044007 INFO (jetty-closer-1662-thread-2) [ ] o.e.j.s.session Stopped scavenging
[junit4] 2> 2044007 ERROR (SUITE-LIROnShardRestartTest-seed#[28E698689983B5BE]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 2044007 INFO (SUITE-LIROnShardRestartTest-seed#[28E698689983B5BE]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:38542 38542
[junit4] 2> 2044231 INFO (Thread-1827) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:38542 38542
[junit4] 2> 2044232 WARN (Thread-1827) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 45 /solr/collections/severalReplicasInLIR/terms/shard1
[junit4] 2> 41 /solr/collections/allReplicasInLIR/terms/shard1
[junit4] 2> 22 /solr/aliases.json
[junit4] 2> 12 /solr/security.json
[junit4] 2> 8 /solr/configs/allReplicasInLIR.AUTOCREATED
[junit4] 2> 8 /solr/configs/allReplicasInLIR.AUTOCREATED/managed-schema
[junit4] 2> 6 /solr/collections/allReplicasInLIR/collectionprops.json
[junit4] 2> 6 /solr/collections/severalReplicasInLIR/collectionprops.json
[junit4] 2> 6 /solr/configs/severalReplicasInLIR.AUTOCREATED
[junit4] 2> 6 /solr/configs/severalReplicasInLIR.AUTOCREATED/managed-schema
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 64 /solr/collections/allReplicasInLIR/state.json
[junit4] 2> 62 /solr/collections/severalReplicasInLIR/state.json
[junit4] 2> 22 /solr/clusterprops.json
[junit4] 2> 22 /solr/clusterstate.json
[junit4] 2> 5 /solr/autoscaling.json
[junit4] 2> 2 /solr/overseer_elect/election/72266798835761192-127.0.0.1:34570_solr-n_0000000009
[junit4] 2> 2 /solr/collections/severalReplicasInLIR/leader_elect/shard1/election/72266798835761169-core_node6-n_0000000005
[junit4] 2> 2 /solr/overseer_elect/election/72266798835761159-127.0.0.1:34281_solr-n_0000000000
[junit4] 2> 2 /solr/collections/severalReplicasInLIR/leader_elect/shard1/election/72266798835761159-core_node5-n_0000000000
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 22 /solr/collections
[junit4] 2> 21 /solr/live_nodes
[junit4] 2> 5 /solr/autoscaling/events/.scheduled_maintenance
[junit4] 2> 5 /solr/autoscaling/events/.auto_add_replicas
[junit4] 2> 5 /solr/overseer/queue-work
[junit4] 2> 5 /solr/overseer/collection-queue-work
[junit4] 2> 4 /solr/overseer/queue
[junit4] 2>
[junit4] 2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.LIROnShardRestartTest_28E698689983B5BE-001
[junit4] 2> Mar 12, 2018 11:41:16 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70), sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@37c94d1d), locale=lv, timezone=Europe/Podgorica
[junit4] 2> NOTE: Linux 4.4.0-112-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=1,free=298832472,total=526385152
[junit4] 2> NOTE: All tests run in this JVM: [SparseHLLTest, TestSystemCollAutoCreate, ZkStateReaderTest, TestCollationFieldDocValues, SolrShardReporterTest, MinimalSchemaTest, TestJettySolrRunner, TestDistributedMap, DocValuesTest, TestJavabinTupleStreamParser, TestInPlaceUpdatesStandalone, BlockJoinFacetRandomTest, PingRequestHandlerTest, BasicDistributedZk2Test, TestCloudJSONFacetJoinDomain, TestLegacyField, TestShortCircuitedRequests, TestUtilizeNode, TestRequestStatusCollectionAPI, AnalyticsQueryTest, TestSchemaVersionResource, SolrCmdDistributorTest, TestSolrCloudWithDelegationTokens, RestartWhileUpdatingTest, PeerSyncWithIndexFingerprintCachingTest, CopyFieldTest, TestFiltering, TestBadConfig, TestTrieFacet, TestQuerySenderListener, DateFieldTest, AnalysisErrorHandlingTest, MultiThreadedOCPTest, TestSolrConfigHandler, ConvertedLegacyTest, ReturnFieldsTest, EnumFieldTest, CdcrRequestHandlerTest, RuleEngineTest, AddReplicaTest, HdfsWriteToMultipleCollectionsTest, TestLMJelinekMercerSimilarityFactory, ParsingFieldUpdateProcessorsTest, TestTolerantSearch, NodeAddedTriggerTest, HdfsRecoveryZkTest, TestConfigSets, TestCloudNestedDocsSort, TestRawTransformer, CoreMergeIndexesAdminHandlerTest, URLClassifyProcessorTest, TestSuggestSpellingConverter, TestManagedStopFilterFactory, ShowFileRequestHandlerTest, LIROnShardRestartTest]
[junit4] Completed [623/785 (2!)] on J2 in 65.35s, 2 tests, 1 error <<< FAILURES!
[...truncated 50010 lines...]
[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1500 - Still
unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1500/
3 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ZkControllerTest
Error Message:
ObjectTracker found 1 object(s) that were not released!!! [Overseer] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.Overseer at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.cloud.Overseer.start(Overseer.java:545) at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:850) at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170) at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135) at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307) at org.apache.solr.cloud.LeaderElector.retryElection(LeaderElector.java:393) at org.apache.solr.cloud.ZkController.rejoinOverseerElection(ZkController.java:2055) at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:331) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [Overseer]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.Overseer
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.cloud.Overseer.start(Overseer.java:545)
at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:850)
at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)
at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)
at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307)
at org.apache.solr.cloud.LeaderElector.retryElection(LeaderElector.java:393)
at org.apache.solr.cloud.ZkController.rejoinOverseerElection(ZkController.java:2055)
at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:331)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([EA2B65CF198D5E35]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertNull(Assert.java:551)
at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:301)
at sun.reflect.GeneratedMethodAccessor36.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:897)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ZkControllerTest
Error Message:
1 thread leaked from SUITE scope at org.apache.solr.cloud.ZkControllerTest: 1) Thread[id=25840, name=OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001, state=RUNNABLE, group=Overseer autoscaling triggers] at java.lang.Throwable.fillInStackTrace(Native Method) at java.lang.Throwable.fillInStackTrace(Throwable.java:783) at java.lang.Throwable.<init>(Throwable.java:250) at java.lang.Exception.<init>(Exception.java:54) at org.apache.zookeeper.KeeperException.<init>(KeeperException.java:439) at org.apache.zookeeper.KeeperException$SessionExpiredException.<init>(KeeperException.java:685) at org.apache.zookeeper.KeeperException.create(KeeperException.java:130) at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215) at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340) at org.apache.solr.common.cloud.SolrZkClient$$Lambda$253/1448267346.execute(Unknown Source) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60) at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340) at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176) at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78) at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:127) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.ZkControllerTest:
1) Thread[id=25840, name=OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001, state=RUNNABLE, group=Overseer autoscaling triggers]
at java.lang.Throwable.fillInStackTrace(Native Method)
at java.lang.Throwable.fillInStackTrace(Throwable.java:783)
at java.lang.Throwable.<init>(Throwable.java:250)
at java.lang.Exception.<init>(Exception.java:54)
at org.apache.zookeeper.KeeperException.<init>(KeeperException.java:439)
at org.apache.zookeeper.KeeperException$SessionExpiredException.<init>(KeeperException.java:685)
at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
at org.apache.solr.common.cloud.SolrZkClient$$Lambda$253/1448267346.execute(Unknown Source)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:127)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([EA2B65CF198D5E35]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ZkControllerTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=25840, name=OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001, state=RUNNABLE, group=Overseer autoscaling triggers] at java.util.Arrays.copyOfRange(Arrays.java:3664) at java.lang.StringBuffer.toString(StringBuffer.java:669) at org.apache.log4j.pattern.NameAbbreviator$PatternAbbreviatorFragment.abbreviate(NameAbbreviator.java:277) at org.apache.log4j.pattern.NameAbbreviator$PatternAbbreviator.abbreviate(NameAbbreviator.java:346) at org.apache.log4j.pattern.NamePatternConverter.abbreviate(NamePatternConverter.java:59) at org.apache.log4j.pattern.LoggerPatternConverter.format(LoggerPatternConverter.java:64) at org.apache.log4j.pattern.BridgePatternConverter.format(BridgePatternConverter.java:119) at org.apache.log4j.EnhancedPatternLayout.format(EnhancedPatternLayout.java:546) at org.apache.log4j.WriterAppender.subAppend(WriterAppender.java:310) at org.apache.log4j.WriterAppender.append(WriterAppender.java:162) at org.apache.log4j.AppenderSkeleton.doAppend(AppenderSkeleton.java:251) at org.apache.log4j.helpers.AppenderAttachableImpl.appendLoopOnAppenders(AppenderAttachableImpl.java:66) at org.apache.log4j.Category.callAppenders(Category.java:206) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.error(Log4jLoggerAdapter.java:576) at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:142) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=25840, name=OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001, state=RUNNABLE, group=Overseer autoscaling triggers]
at java.util.Arrays.copyOfRange(Arrays.java:3664)
at java.lang.StringBuffer.toString(StringBuffer.java:669)
at org.apache.log4j.pattern.NameAbbreviator$PatternAbbreviatorFragment.abbreviate(NameAbbreviator.java:277)
at org.apache.log4j.pattern.NameAbbreviator$PatternAbbreviator.abbreviate(NameAbbreviator.java:346)
at org.apache.log4j.pattern.NamePatternConverter.abbreviate(NamePatternConverter.java:59)
at org.apache.log4j.pattern.LoggerPatternConverter.format(LoggerPatternConverter.java:64)
at org.apache.log4j.pattern.BridgePatternConverter.format(BridgePatternConverter.java:119)
at org.apache.log4j.EnhancedPatternLayout.format(EnhancedPatternLayout.java:546)
at org.apache.log4j.WriterAppender.subAppend(WriterAppender.java:310)
at org.apache.log4j.WriterAppender.append(WriterAppender.java:162)
at org.apache.log4j.AppenderSkeleton.doAppend(AppenderSkeleton.java:251)
at org.apache.log4j.helpers.AppenderAttachableImpl.appendLoopOnAppenders(AppenderAttachableImpl.java:66)
at org.apache.log4j.Category.callAppenders(Category.java:206)
at org.apache.log4j.Category.forcedLog(Category.java:391)
at org.apache.log4j.Category.log(Category.java:856)
at org.slf4j.impl.Log4jLoggerAdapter.error(Log4jLoggerAdapter.java:576)
at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:142)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([EA2B65CF198D5E35]:0)
Build Log:
[...truncated 14049 lines...]
[junit4] Suite: org.apache.solr.cloud.ZkControllerTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ZkControllerTest_EA2B65CF198D5E35-001/init-core-data-001
[junit4] 2> 2375577 WARN (SUITE-ZkControllerTest-seed#[EA2B65CF198D5E35]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=23 numCloses=23
[junit4] 2> 2375578 INFO (SUITE-ZkControllerTest-seed#[EA2B65CF198D5E35]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 2375579 INFO (SUITE-ZkControllerTest-seed#[EA2B65CF198D5E35]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=None)
[junit4] 2> 2375580 INFO (SUITE-ZkControllerTest-seed#[EA2B65CF198D5E35]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 2375585 INFO (TEST-ZkControllerTest.testGetHostName-seed#[EA2B65CF198D5E35]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testGetHostName
[junit4] 2> 2375586 INFO (TEST-ZkControllerTest.testGetHostName-seed#[EA2B65CF198D5E35]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 2375586 INFO (Thread-9874) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 2375586 INFO (Thread-9874) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 2375589 ERROR (Thread-9874) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 2375686 INFO (TEST-ZkControllerTest.testGetHostName-seed#[EA2B65CF198D5E35]) [ ] o.a.s.c.ZkTestServer start zk server on port:43360
[junit4] 2> 2375692 INFO (zkConnectionManagerCallback-6040-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2375694 INFO (zkConnectionManagerCallback-6042-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2375703 INFO (TEST-ZkControllerTest.testGetHostName-seed#[EA2B65CF198D5E35]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@644cf124, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 2375708 INFO (zkConnectionManagerCallback-6046-thread-1-processing-n:127.0.0.1:8983_solr) [n:127.0.0.1:8983_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2375828 INFO (TEST-ZkControllerTest.testGetHostName-seed#[EA2B65CF198D5E35]) [n:127.0.0.1:8983_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2375829 INFO (TEST-ZkControllerTest.testGetHostName-seed#[EA2B65CF198D5E35]) [n:127.0.0.1:8983_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:8983_solr
[junit4] 2> 2375830 INFO (TEST-ZkControllerTest.testGetHostName-seed#[EA2B65CF198D5E35]) [n:127.0.0.1:8983_solr ] o.a.s.c.Overseer Overseer (id=73434860240437250-127.0.0.1:8983_solr-n_0000000000) starting
[junit4] 2> 2375840 INFO (zkConnectionManagerCallback-6051-thread-1-processing-n:127.0.0.1:8983_solr) [n:127.0.0.1:8983_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2375842 INFO (TEST-ZkControllerTest.testGetHostName-seed#[EA2B65CF198D5E35]) [n:127.0.0.1:8983_solr ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:43360/solr ready
[junit4] 2> 2375843 INFO (TEST-ZkControllerTest.testGetHostName-seed#[EA2B65CF198D5E35]) [n:127.0.0.1:8983_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:8983_solr
[junit4] 2> 2375844 INFO (TEST-ZkControllerTest.testGetHostName-seed#[EA2B65CF198D5E35]) [n:127.0.0.1:8983_solr ] o.a.s.c.Overseer Overseer (id=73434860240437250-127.0.0.1:8983_solr-n_0000000000) closing
[junit4] 2> 2375845 INFO (OverseerStateUpdate-73434860240437250-127.0.0.1:8983_solr-n_0000000000) [n:127.0.0.1:8983_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2375846 INFO (OverseerStateUpdate-73434860240437250-127.0.0.1:8983_solr-n_0000000000) [n:127.0.0.1:8983_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:8983_solr
[junit4] 2> 2375847 WARN (OverseerExitThread) [n:127.0.0.1:8983_solr ] o.a.s.c.Overseer I'm exiting, but I'm still the leader
[junit4] 2> 2375849 WARN (OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000000) [n:127.0.0.1:8983_solr ] o.a.s.c.a.OverseerTriggerThread Interrupted
[junit4] 2> java.lang.InterruptedException
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1220)
[junit4] 2> at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:165)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 2375852 INFO (OverseerExitThread) [n:127.0.0.1:8983_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:8983_solr
[junit4] 2> 2375853 INFO (OverseerExitThread) [n:127.0.0.1:8983_solr ] o.a.s.c.Overseer Overseer (id=73434860240437250-127.0.0.1:8983_solr-n_0000000001) starting
[junit4] 2> 2375860 ERROR (OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001) [n:127.0.0.1:8983_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:127)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 2375861 ERROR (OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001) [n:127.0.0.1:8983_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:127)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 2375861 ERROR (OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001) [n:127.0.0.1:8983_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:127)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 2375862 ERROR (OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001) [n:127.0.0.1:8983_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:127)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 2375862 ERROR (OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001) [n:127.0.0.1:8983_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:127)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 2375863 ERROR (OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001) [n:127.0.0.1:8983_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:127)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 2375863 ERROR (OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001) [n:127.0.0.1:8983_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:127)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 2375863 ERROR (OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001) [n:127.0.0.1:8983_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:127)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 2375863 ERROR (OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001) [n:127.0.0.1:8983_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:127)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 2375864 ERROR (OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001) [n:127.0.0.1:8983_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:127)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 2375864 ERROR (OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001) [n:127.0.0.1:8983_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:127)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 2375864 ERROR (OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001) [n:127.0.0.1:8983_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:127)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 2375864 ERROR (OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001) [n:127.0.0.1:8983_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:127)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 2375865 ERROR (OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001) [n:127.0.0.1:8983_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:127)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 2375865 ERROR (OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001) [n:127.0.0.1:8983_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:127)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 2375865 ERROR (OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001) [n:127.0.0.1:8983_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:127)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 2375865 ERROR (OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001) [n:127.0.0.1:8983_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:127)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 2375866 ERROR (OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001) [n:127.0.0.1:8983_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:127)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 2375866 ERROR (OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001) [n:127.0.0.1:8983_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:127)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 2375866 ERROR (OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001) [n:127.0.0.1:8983_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:127)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 2375867 ERROR (OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001) [n:127.0.0.1:8983_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:127)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 2375867 ERROR (OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001) [n:127.0.0.1:8983_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:127)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 2375867 ERROR (OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001) [n:127.0.0.1:8983_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:127)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 2375868 ERROR (OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001) [n:127.0.0.1:8983_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:127)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 2375868 ERROR (OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001) [n:127.0.0.1:8983_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:127)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 2375868 ERROR (OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001) [n:127.0.0.1:8983_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:127)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 2375869 ERROR (OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001) [n:127.0.0.1:8983_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:127)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 2375869 ERROR (OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001) [n:127.0.0.1:8983_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:127)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 2375869 ERROR (OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001) [n:127.0.0.1:8983_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:127)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 2375869 ERROR (OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001) [n:127.0.0.1:8983_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:127)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 2375869 ERROR (OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001) [n:127.0.0.1:8983_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:127)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 2375870 ERROR (OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001) [n:127.0.0.1:8983_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:127)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 2375870 ERROR (OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001) [n:127.0.0.1:8983_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:127)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 2375870 ERROR (OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001) [n:127.0.0.1:8983_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:127)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 2375870 ERROR (OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001) [n:127.0.0.1:8983_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:127)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 2375871 ERROR (OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001) [n:127.0.0.1:8983_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:127)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 2375871 ERROR (OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001) [n:127.0.0.1:8983_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:127)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 2375871 ERROR (OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001) [n:127.0.0.1:8983_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:127)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 2375871 ERROR (OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001) [n:127.0.0.1:8983_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:127)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 2375872 ERROR (OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001) [n:127.0.0.1:8983_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:127)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 2375872 ERROR (OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001) [n:127.0.0.1:8983_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:127)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 2375872 ERROR (OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001) [n:127.0.0.1:8983_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:127)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 2375873 ERROR (OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001) [n:127.0.0.1:8983_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:127)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 2375873 ERROR (OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001) [n:127.0.0.1:8983_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:127)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4]
[...truncated too long message...]
kClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> 2483992 ERROR (OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001) [n:127.0.0.1:8983_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:127)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=ZkControllerTest -Dtests.seed=EA2B65CF198D5E35 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=ms -Dtests.timezone=America/Cordoba -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] 2> 2483992 ERROR (OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001) [n:127.0.0.1:8983_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:127)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] ERROR 0.00s J1 | ZkControllerTest (suite) <<<
[junit4] > Throwable #1: java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [Overseer]
[junit4] > org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.Overseer
[junit4] > at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
[junit4] > at org.apache.solr.cloud.Overseer.start(Overseer.java:545)
[junit4] > at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:850)
[junit4] > at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)
[junit4] > at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)
[junit4] > at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307)
[junit4] > at org.apache.solr.cloud.LeaderElector.retryElection(LeaderElector.java:393)
[junit4] > at org.apache.solr.cloud.ZkController.rejoinOverseerElection(ZkController.java:2055)
[junit4] > at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:331)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > at __randomizedtesting.SeedInfo.seed([EA2B65CF198D5E35]:0)
[junit4] > at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:301)
[junit4] > at java.lang.Thread.run(Thread.java:748)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.ZkControllerTest:
[junit4] > 1) Thread[id=25840, name=OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001, state=RUNNABLE, group=Overseer autoscaling triggers]
[junit4] > at java.lang.Throwable.fillInStackTrace(Native Method)
[junit4] > at java.lang.Throwable.fillInStackTrace(Throwable.java:783)
[junit4] > at java.lang.Throwable.<init>(Throwable.java:250)
[junit4] > at java.lang.Exception.<init>(Exception.java:54)
[junit4] > at org.apache.zookeeper.KeeperException.<init>(KeeperException.java:439)
[junit4] > at org.apache.zookeeper.KeeperException$SessionExpiredException.<init>(KeeperException.java:685)
[junit4] > at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] > at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] > at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] > at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] > at org.apache.solr.common.cloud.SolrZkClient$$Lambda$253/1448267346.execute(Unknown Source)
[junit4] > at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] > at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] > at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] > at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] > at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:127)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > at __randomizedtesting.SeedInfo.seed([EA2B65CF198D5E35]:0)Throwable #3: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
[junit4] > 1) Thread[id=25840, name=OverseerAutoScalingTriggerThread-73434860240437250-127.0.0.1:8983_solr-n_0000000001, state=RUNNABLE, group=Overseer autoscaling triggers]
[junit4] > at java.util.Arrays.copyOfRange(Arrays.java:3664)
[junit4] > at java.lang.StringBuffer.toString(StringBuffer.java:669)
[junit4] > at org.apache.log4j.pattern.NameAbbreviator$PatternAbbreviatorFragment.abbreviate(NameAbbreviator.java:277)
[junit4] > at org.apache.log4j.pattern.NameAbbreviator$PatternAbbreviator.abbreviate(NameAbbreviator.java:346)
[junit4] > at org.apache.log4j.pattern.NamePatternConverter.abbreviate(NamePatternConverter.java:59)
[junit4] > at org.apache.log4j.pattern.LoggerPatternConverter.format(LoggerPatternConverter.java:64)
[junit4] > at org.apache.log4j.pattern.BridgePatternConverter.format(BridgePatternConverter.java:119)
[junit4] > at org.apache.log4j.EnhancedPatternLayout.format(EnhancedPatternLayout.java:546)
[junit4] > at org.apache.log4j.WriterAppender.subAppend(WriterAppender.java:310)
[junit4] > at org.apache.log4j.WriterAppender.append(WriterAppender.java:162)
[junit4] > at org.apache.log4j.AppenderSkeleton.doAppend(AppenderSkeleton.java:251)
[junit4] > at org.apache.log4j.helpers.AppenderAttachableImpl.appendLoopOnAppenders(AppenderAttachableImpl.java:66)
[junit4] > at org.apache.log4j.Category.callAppenders(Category.java:206)
[junit4] > at org.apache.log4j.Category.forcedLog(Category.java:391)
[junit4] > at org.apache.log4j.Category.log(Category.java:856)
[junit4] > at org.slf4j.impl.Log4jLoggerAdapter.error(Log4jLoggerAdapter.java:576)
[junit4] > at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:142)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > at __randomizedtesting.SeedInfo.seed([EA2B65CF198D5E35]:0)
[junit4] Completed [405/785 (1!)] on J1 in 108.43s, 4 tests, 1 failure, 2 errors, 1 skipped <<< FAILURES!
[...truncated 50701 lines...]
[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1499 - Failure
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1499/
1 tests failed.
FAILED: org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test
Error Message:
Java heap space
Stack Trace:
java.lang.OutOfMemoryError: Java heap space
at __randomizedtesting.SeedInfo.seed([1D9FE4F5C38DA91E:95CBDB2F6D71C4E6]:0)
at java.util.Arrays.copyOf(Arrays.java:3332)
at java.lang.AbstractStringBuilder.ensureCapacityInternal(AbstractStringBuilder.java:124)
at java.lang.AbstractStringBuilder.append(AbstractStringBuilder.java:649)
at java.lang.StringBuilder.append(StringBuilder.java:202)
at org.apache.http.client.utils.URLEncodedUtils.urlEncode(URLEncodedUtils.java:536)
at org.apache.http.client.utils.URLEncodedUtils.encodeFormFields(URLEncodedUtils.java:652)
at org.apache.http.client.utils.URLEncodedUtils.format(URLEncodedUtils.java:404)
at org.apache.http.client.utils.URLEncodedUtils.format(URLEncodedUtils.java:382)
at org.apache.http.client.entity.UrlEncodedFormEntity.<init>(UrlEncodedFormEntity.java:75)
at org.apache.solr.client.solrj.impl.HttpSolrClient.fillContentStream(HttpSolrClient.java:513)
at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:420)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1104)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:884)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:817)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:974)
at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:990)
at org.apache.solr.cloud.CloudInspectUtil.compareResults(CloudInspectUtil.java:228)
at org.apache.solr.cloud.CloudInspectUtil.compareResults(CloudInspectUtil.java:167)
at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.testIndexingBatchPerRequestWithHttpSolrClient(FullSolrCloudDistribCmdsTest.java:668)
at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test(FullSolrCloudDistribCmdsTest.java:152)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
Build Log:
[...truncated 14127 lines...]
[junit4] Suite: org.apache.solr.cloud.FullSolrCloudDistribCmdsTest
[junit4] 2> 232707 INFO (SUITE-FullSolrCloudDistribCmdsTest-seed#[1D9FE4F5C38DA91E]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_1D9FE4F5C38DA91E-001/init-core-data-001
[junit4] 2> 232709 WARN (SUITE-FullSolrCloudDistribCmdsTest-seed#[1D9FE4F5C38DA91E]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=612 numCloses=612
[junit4] 2> 232709 INFO (SUITE-FullSolrCloudDistribCmdsTest-seed#[1D9FE4F5C38DA91E]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 232711 INFO (SUITE-FullSolrCloudDistribCmdsTest-seed#[1D9FE4F5C38DA91E]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
[junit4] 2> 232711 INFO (SUITE-FullSolrCloudDistribCmdsTest-seed#[1D9FE4F5C38DA91E]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
[junit4] 2> 232721 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 232727 INFO (Thread-457) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 232727 INFO (Thread-457) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 232741 ERROR (Thread-457) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 232827 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.ZkTestServer start zk server on port:40476
[junit4] 2> 232845 INFO (zkConnectionManagerCallback-381-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 232857 INFO (zkConnectionManagerCallback-383-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 232863 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 232866 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4] 2> 232867 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 232869 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 232871 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 232873 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 232874 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 232876 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 232877 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 232878 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 232880 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 232882 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
[junit4] 2> 233191 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T08:27:37+11:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 233282 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 233282 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 233282 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.e.j.s.session Scavenging every 660000ms
[junit4] 2> 233291 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3b11e727{/,null,AVAILABLE}
[junit4] 2> 233294 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@3edca860{HTTP/1.1,[http/1.1]}{127.0.0.1:56858}
[junit4] 2> 233295 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.e.j.s.Server Started @236385ms
[junit4] 2> 233295 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_1D9FE4F5C38DA91E-001/tempDir-001/control/data, hostContext=/, hostPort=56858, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_1D9FE4F5C38DA91E-001/control-001/cores}
[junit4] 2> 233295 ERROR (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 233295 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 8.0.0
[junit4] 2> 233295 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 233295 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 233295 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-03-10T05:58:10.041Z
[junit4] 2> 233315 INFO (zkConnectionManagerCallback-385-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 233317 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 233317 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_1D9FE4F5C38DA91E-001/control-001/solr.xml
[junit4] 2> 233322 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 233322 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 233323 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@4c47c964, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 233338 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40476/solr
[junit4] 2> 233344 INFO (zkConnectionManagerCallback-389-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 233373 INFO (zkConnectionManagerCallback-391-thread-1-processing-n:127.0.0.1:56858_) [n:127.0.0.1:56858_ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 233547 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [n:127.0.0.1:56858_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 233547 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [n:127.0.0.1:56858_ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:56858_
[junit4] 2> 233551 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [n:127.0.0.1:56858_ ] o.a.s.c.Overseer Overseer (id=73428174100889604-127.0.0.1:56858_-n_0000000000) starting
[junit4] 2> 233565 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [n:127.0.0.1:56858_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:56858_
[junit4] 2> 233586 INFO (OverseerStateUpdate-73428174100889604-127.0.0.1:56858_-n_0000000000) [n:127.0.0.1:56858_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 233668 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [n:127.0.0.1:56858_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4c47c964
[junit4] 2> 233680 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [n:127.0.0.1:56858_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4c47c964
[junit4] 2> 233680 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [n:127.0.0.1:56858_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4c47c964
[junit4] 2> 233830 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [n:127.0.0.1:56858_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_1D9FE4F5C38DA91E-001/control-001/cores
[junit4] 2> 233857 INFO (zkConnectionManagerCallback-397-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 233858 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 233859 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40476/solr ready
[junit4] 2> 233862 INFO (qtp454547604-1811) [n:127.0.0.1:56858_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:56858_&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 233866 INFO (OverseerThreadFactory-552-thread-1-processing-n:127.0.0.1:56858_) [n:127.0.0.1:56858_ ] o.a.s.c.a.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 233978 INFO (qtp454547604-1815) [n:127.0.0.1:56858_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 233979 INFO (qtp454547604-1815) [n:127.0.0.1:56858_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 234085 INFO (zkCallback-390-thread-1-processing-n:127.0.0.1:56858_) [n:127.0.0.1:56858_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 234999 INFO (qtp454547604-1815) [n:127.0.0.1:56858_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 235014 INFO (qtp454547604-1815) [n:127.0.0.1:56858_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 235107 INFO (qtp454547604-1815) [n:127.0.0.1:56858_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 235153 INFO (qtp454547604-1815) [n:127.0.0.1:56858_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 235154 INFO (qtp454547604-1815) [n:127.0.0.1:56858_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4c47c964
[junit4] 2> 235154 INFO (qtp454547604-1815) [n:127.0.0.1:56858_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 235154 INFO (qtp454547604-1815) [n:127.0.0.1:56858_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_1D9FE4F5C38DA91E-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_1D9FE4F5C38DA91E-001/control-001/cores/control_collection_shard1_replica_n1/data/]
[junit4] 2> 235160 INFO (qtp454547604-1815) [n:127.0.0.1:56858_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=15, maxMergeAtOnceExplicit=30, maxMergedSegmentMB=59.888671875, floorSegmentMB=0.5185546875, forceMergeDeletesPctAllowed=18.31411628818963, segmentsPerTier=16.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.3948930594730886
[junit4] 2> 235167 WARN (qtp454547604-1815) [n:127.0.0.1:56858_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 235232 INFO (qtp454547604-1815) [n:127.0.0.1:56858_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 235232 INFO (qtp454547604-1815) [n:127.0.0.1:56858_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 235234 INFO (qtp454547604-1815) [n:127.0.0.1:56858_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 235234 INFO (qtp454547604-1815) [n:127.0.0.1:56858_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 235236 INFO (qtp454547604-1815) [n:127.0.0.1:56858_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=14, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 235237 INFO (qtp454547604-1815) [n:127.0.0.1:56858_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@57e47328[control_collection_shard1_replica_n1] main]
[junit4] 2> 235239 INFO (qtp454547604-1815) [n:127.0.0.1:56858_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 235240 INFO (qtp454547604-1815) [n:127.0.0.1:56858_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 235242 INFO (qtp454547604-1815) [n:127.0.0.1:56858_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 235243 INFO (searcherExecutor-555-thread-1-processing-n:127.0.0.1:56858_ x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:56858_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@57e47328[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 235243 INFO (qtp454547604-1815) [n:127.0.0.1:56858_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1594529144623857664
[junit4] 2> 235250 INFO (qtp454547604-1815) [n:127.0.0.1:56858_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkShardTerms Successful update terms at /collections/control_collection/terms/shard1 to Terms{values={core_node2=0}, version=0}
[junit4] 2> 235254 INFO (qtp454547604-1815) [n:127.0.0.1:56858_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 235254 INFO (qtp454547604-1815) [n:127.0.0.1:56858_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 235255 INFO (qtp454547604-1815) [n:127.0.0.1:56858_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:56858/control_collection_shard1_replica_n1/
[junit4] 2> 235255 INFO (qtp454547604-1815) [n:127.0.0.1:56858_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 235255 INFO (qtp454547604-1815) [n:127.0.0.1:56858_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:56858/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 235255 INFO (qtp454547604-1815) [n:127.0.0.1:56858_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 235267 INFO (qtp454547604-1815) [n:127.0.0.1:56858_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:56858/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 235369 INFO (zkCallback-390-thread-1-processing-n:127.0.0.1:56858_) [n:127.0.0.1:56858_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 235418 INFO (qtp454547604-1815) [n:127.0.0.1:56858_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 235421 INFO (qtp454547604-1815) [n:127.0.0.1:56858_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1442
[junit4] 2> 235429 INFO (qtp454547604-1811) [n:127.0.0.1:56858_ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 235522 INFO (zkCallback-390-thread-1-processing-n:127.0.0.1:56858_) [n:127.0.0.1:56858_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 235870 INFO (OverseerCollectionConfigSetProcessor-73428174100889604-127.0.0.1:56858_-n_0000000000) [n:127.0.0.1:56858_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 236429 INFO (qtp454547604-1811) [n:127.0.0.1:56858_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:56858_&wt=javabin&version=2} status=0 QTime=2567
[junit4] 2> 236435 INFO (zkConnectionManagerCallback-401-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 236437 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 236438 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40476/solr ready
[junit4] 2> 236438 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 236439 INFO (qtp454547604-1815) [n:127.0.0.1:56858_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=3&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 236445 INFO (OverseerThreadFactory-552-thread-2-processing-n:127.0.0.1:56858_) [n:127.0.0.1:56858_ ] o.a.s.c.a.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 236447 WARN (OverseerThreadFactory-552-thread-2-processing-n:127.0.0.1:56858_) [n:127.0.0.1:56858_ ] o.a.s.c.a.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 236656 INFO (qtp454547604-1815) [n:127.0.0.1:56858_ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 236656 INFO (qtp454547604-1815) [n:127.0.0.1:56858_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=3&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=216
[junit4] 2> 236792 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_1D9FE4F5C38DA91E-001/shard-1-001 of type NRT
[junit4] 2> 236792 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T08:27:37+11:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 236793 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 236793 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 236793 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.e.j.s.session Scavenging every 600000ms
[junit4] 2> 236794 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3dd426b9{/,null,AVAILABLE}
[junit4] 2> 236794 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@24293fe7{HTTP/1.1,[http/1.1]}{127.0.0.1:50070}
[junit4] 2> 236795 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.e.j.s.Server Started @239885ms
[junit4] 2> 236795 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_1D9FE4F5C38DA91E-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/, hostPort=50070, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_1D9FE4F5C38DA91E-001/shard-1-001/cores}
[junit4] 2> 236795 ERROR (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 236795 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 8.0.0
[junit4] 2> 236795 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 236795 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 236795 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-03-10T05:58:13.541Z
[junit4] 2> 236799 INFO (zkConnectionManagerCallback-403-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 236800 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 236800 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_1D9FE4F5C38DA91E-001/shard-1-001/solr.xml
[junit4] 2> 236805 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 236805 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 236806 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@4c47c964, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 236809 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40476/solr
[junit4] 2> 236811 INFO (zkConnectionManagerCallback-407-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 236814 INFO (zkConnectionManagerCallback-409-thread-1-processing-n:127.0.0.1:50070_) [n:127.0.0.1:50070_ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 236824 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [n:127.0.0.1:50070_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 236827 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [n:127.0.0.1:50070_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 236830 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [n:127.0.0.1:50070_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50070_
[junit4] 2> 236832 INFO (zkCallback-400-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 236832 INFO (zkCallback-390-thread-1-processing-n:127.0.0.1:56858_) [n:127.0.0.1:56858_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 236837 INFO (zkCallback-408-thread-1-processing-n:127.0.0.1:50070_) [n:127.0.0.1:50070_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 236935 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [n:127.0.0.1:50070_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4c47c964
[junit4] 2> 236948 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [n:127.0.0.1:50070_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4c47c964
[junit4] 2> 236948 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [n:127.0.0.1:50070_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4c47c964
[junit4] 2> 236950 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [n:127.0.0.1:50070_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_1D9FE4F5C38DA91E-001/shard-1-001/cores
[junit4] 2> 236991 INFO (qtp1161713663-1866) [n:127.0.0.1:50070_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:50070_&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 236997 INFO (OverseerCollectionConfigSetProcessor-73428174100889604-127.0.0.1:56858_-n_0000000000) [n:127.0.0.1:56858_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 236998 INFO (OverseerThreadFactory-552-thread-3-processing-n:127.0.0.1:56858_) [n:127.0.0.1:56858_ ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:50070_ for creating new replica
[junit4] 2> 237004 INFO (qtp1161713663-1864) [n:127.0.0.1:50070_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n61&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 237004 INFO (qtp1161713663-1864) [n:127.0.0.1:50070_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 238030 INFO (qtp1161713663-1864) [n:127.0.0.1:50070_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 238045 INFO (qtp1161713663-1864) [n:127.0.0.1:50070_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.s.IndexSchema [collection1_shard2_replica_n61] Schema name=test
[junit4] 2> 238142 INFO (qtp1161713663-1864) [n:127.0.0.1:50070_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 238170 INFO (qtp1161713663-1864) [n:127.0.0.1:50070_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_n61' using configuration from collection collection1, trusted=true
[junit4] 2> 238170 INFO (qtp1161713663-1864) [n:127.0.0.1:50070_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_n61' (registry 'solr.core.collection1.shard2.replica_n61') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4c47c964
[junit4] 2> 238170 INFO (qtp1161713663-1864) [n:127.0.0.1:50070_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 238171 INFO (qtp1161713663-1864) [n:127.0.0.1:50070_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.SolrCore [[collection1_shard2_replica_n61] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_1D9FE4F5C38DA91E-001/shard-1-001/cores/collection1_shard2_replica_n61], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_1D9FE4F5C38DA91E-001/shard-1-001/cores/collection1_shard2_replica_n61/data/]
[junit4] 2> 238174 INFO (qtp1161713663-1864) [n:127.0.0.1:50070_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=15, maxMergeAtOnceExplicit=30, maxMergedSegmentMB=59.888671875, floorSegmentMB=0.5185546875, forceMergeDeletesPctAllowed=18.31411628818963, segmentsPerTier=16.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.3948930594730886
[junit4] 2> 238179 WARN (qtp1161713663-1864) [n:127.0.0.1:50070_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 238256 INFO (qtp1161713663-1864) [n:127.0.0.1:50070_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 238256 INFO (qtp1161713663-1864) [n:127.0.0.1:50070_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 238258 INFO (qtp1161713663-1864) [n:127.0.0.1:50070_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 238258 INFO (qtp1161713663-1864) [n:127.0.0.1:50070_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 238259 INFO (qtp1161713663-1864) [n:127.0.0.1:50070_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=14, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 238260 INFO (qtp1161713663-1864) [n:127.0.0.1:50070_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.s.SolrIndexSearcher Opening [Searcher@30ccfd14[collection1_shard2_replica_n61] main]
[junit4] 2> 238261 INFO (qtp1161713663-1864) [n:127.0.0.1:50070_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 238262 INFO (qtp1161713663-1864) [n:127.0.0.1:50070_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 238263 INFO (qtp1161713663-1864) [n:127.0.0.1:50070_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 238266 INFO (searcherExecutor-566-thread-1-processing-n:127.0.0.1:50070_ x:collection1_shard2_replica_n61 s:shard2 c:collection1) [n:127.0.0.1:50070_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.SolrCore [collection1_shard2_replica_n61] Registered new searcher Searcher@30ccfd14[collection1_shard2_replica_n61] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 238266 INFO (qtp1161713663-1864) [n:127.0.0.1:50070_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1594529147793702912
[junit4] 2> 238272 INFO (qtp1161713663-1864) [n:127.0.0.1:50070_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.ZkShardTerms Successful update terms at /collections/collection1/terms/shard2 to Terms{values={core_node62=0}, version=0}
[junit4] 2> 238275 INFO (qtp1161713663-1864) [n:127.0.0.1:50070_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 238275 INFO (qtp1161713663-1864) [n:127.0.0.1:50070_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 238275 INFO (qtp1161713663-1864) [n:127.0.0.1:50070_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:50070/collection1_shard2_replica_n61/
[junit4] 2> 238276 INFO (qtp1161713663-1864) [n:127.0.0.1:50070_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 238276 INFO (qtp1161713663-1864) [n:127.0.0.1:50070_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.SyncStrategy http://127.0.0.1:50070/collection1_shard2_replica_n61/ has no replicas
[junit4] 2> 238276 INFO (qtp1161713663-1864) [n:127.0.0.1:50070_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 238280 INFO (qtp1161713663-1864) [n:127.0.0.1:50070_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:50070/collection1_shard2_replica_n61/ shard2
[junit4] 2> 238431 INFO (qtp1161713663-1864) [n:127.0.0.1:50070_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 238434 INFO (qtp1161713663-1864) [n:127.0.0.1:50070_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n61&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1429
[junit4] 2> 238439 INFO (qtp1161713663-1866) [n:127.0.0.1:50070_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:50070_&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2} status=0 QTime=1448
[junit4] 2> 238611 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_1D9FE4F5C38DA91E-001/shard-2-001 of type NRT
[junit4] 2> 238612 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T08:27:37+11:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 238613 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 238613 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 238613 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.e.j.s.session Scavenging every 600000ms
[junit4] 2> 238614 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@412f69f5{/,null,AVAILABLE}
[junit4] 2> 238614 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@293238eb{HTTP/1.1,[http/1.1]}{127.0.0.1:35234}
[junit4] 2> 238614 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.e.j.s.Server Started @241705ms
[junit4] 2> 238614 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_1D9FE4F5C38DA91E-001/tempDir-001/jetty2, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/, hostPort=35234, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_1D9FE4F5C38DA91E-001/shard-2-001/cores}
[junit4] 2> 238615 ERROR (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 238616 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 8.0.0
[junit4] 2> 238616 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 238616 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 238616 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-03-10T05:58:15.362Z
[junit4] 2> 238631 INFO (zkConnectionManagerCallback-413-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 238633 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 238633 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_1D9FE4F5C38DA91E-001/shard-2-001/solr.xml
[junit4] 2> 238639 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 238639 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 238640 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@4c47c964, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 238649 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40476/solr
[junit4] 2> 238651 INFO (zkConnectionManagerCallback-417-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 238655 INFO (zkConnectionManagerCallback-419-thread-1-processing-n:127.0.0.1:35234_) [n:127.0.0.1:35234_ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 238670 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [n:127.0.0.1:35234_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 238673 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [n:127.0.0.1:35234_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 238675 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [n:127.0.0.1:35234_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35234_
[junit4] 2> 238680 INFO (zkCallback-400-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 238682 INFO (zkCallback-390-thread-1-processing-n:127.0.0.1:56858_) [n:127.0.0.1:56858_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 238684 INFO (zkCallback-408-thread-1-processing-n:127.0.0.1:50070_) [n:127.0.0.1:50070_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 238684 INFO (zkCallback-418-thread-1-processing-n:127.0.0.1:35234_) [n:127.0.0.1:35234_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 238799 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [n:127.0.0.1:35234_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4c47c964
[junit4] 2> 238813 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [n:127.0.0.1:35234_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4c47c964
[junit4] 2> 238813 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [n:127.0.0.1:35234_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4c47c964
[junit4] 2> 238826 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [n:127.0.0.1:35234_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_1D9FE4F5C38DA91E-001/shard-2-001/cores
[junit4] 2> 238882 INFO (qtp2132072037-1891) [n:127.0.0.1:35234_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:35234_&action=ADDREPLICA&collection=collection1&shard=shard3&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 238885 INFO (OverseerCollectionConfigSetProcessor-73428174100889604-127.0.0.1:56858_-n_0000000000) [n:127.0.0.1:56858_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 238887 INFO (OverseerThreadFactory-552-thread-4-processing-n:127.0.0.1:56858_) [n:127.0.0.1:56858_ ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:35234_ for creating new replica
[junit4] 2> 238890 INFO (qtp2132072037-1896) [n:127.0.0.1:35234_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard3_replica_n63&action=CREATE&collection=collection1&shard=shard3&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 238890 INFO (qtp2132072037-1896) [n:127.0.0.1:35234_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 239911 INFO (qtp2132072037-1896) [n:127.0.0.1:35234_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 239929 INFO (qtp2132072037-1896) [n:127.0.0.1:35234_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.s.IndexSchema [collection1_shard3_replica_n63] Schema name=test
[junit4] 2> 240023 INFO (qtp2132072037-1896) [n:127.0.0.1:35234_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 240060 INFO (qtp2132072037-1896) [n:127.0.0.1:35234_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard3_replica_n63' using configuration from collection collection1, trusted=true
[junit4] 2> 240061 INFO (qtp2132072037-1896) [n:127.0.0.1:35234_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard3.replica_n63' (registry 'solr.core.collection1.shard3.replica_n63') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4c47c964
[junit4] 2> 240061 INFO (qtp2132072037-1896) [n:127.0.0.1:35234_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 240061 INFO (qtp2132072037-1896) [n:127.0.0.1:35234_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.c.SolrCore [[collection1_shard3_replica_n63] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_1D9FE4F5C38DA91E-001/shard-2-001/cores/collection1_shard3_replica_n63], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_1D9FE4F5C38DA91E-001/shard-2-001/cores/collection1_shard3_replica_n63/data/]
[junit4] 2> 240064 INFO (qtp2132072037-1896) [n:127.0.0.1:35234_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=15, maxMergeAtOnceExplicit=30, maxMergedSegmentMB=59.888671875, floorSegmentMB=0.5185546875, forceMergeDeletesPctAllowed=18.31411628818963, segmentsPerTier=16.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.3948930594730886
[junit4] 2> 240068 WARN (qtp2132072037-1896) [n:127.0.0.1:35234_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 240127 INFO (qtp2132072037-1896) [n:127.0.0.1:35234_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 240127 INFO (qtp2132072037-1896) [n:127.0.0.1:35234_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 240129 INFO (qtp2132072037-1896) [n:127.0.0.1:35234_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 240129 INFO (qtp2132072037-1896) [n:127.0.0.1:35234_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 240130 INFO (qtp2132072037-1896) [n:127.0.0.1:35234_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=14, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 240131 INFO (qtp2132072037-1896) [n:127.0.0.1:35234_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.s.SolrIndexSearcher Opening [Searcher@3c502542[collection1_shard3_replica_n63] main]
[junit4] 2> 240133 INFO (qtp2132072037-1896) [n:127.0.0.1:35234_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 240133 INFO (qtp2132072037-1896) [n:127.0.0.1:35234_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 240134 INFO (qtp2132072037-1896) [n:127.0.0.1:35234_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 240134 INFO (qtp2132072037-1896) [n:127.0.0.1:35234_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1594529149752442880
[junit4] 2> 240138 INFO (searcherExecutor-577-thread-1-processing-n:127.0.0.1:35234_ x:collection1_shard3_replica_n63 s:shard3 c:collection1) [n:127.0.0.1:35234_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.c.SolrCore [collection1_shard3_replica_n63] Registered new searcher Searcher@3c502542[collection1_shard3_replica_n63] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 240141 INFO (qtp2132072037-1896) [n:127.0.0.1:35234_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.c.ZkShardTerms Successful update terms at /collections/collection1/terms/shard3 to Terms{values={core_node64=0}, version=0}
[junit4] 2> 240144 INFO (qtp2132072037-1896) [n:127.0.0.1:35234_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 240144 INFO (qtp2132072037-1896) [n:127.0.0.1:35234_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 240144 INFO (qtp2132072037-1896) [n:127.0.0.1:35234_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:35234/collection1_shard3_replica_n63/
[junit4] 2> 240144 INFO (qtp2132072037-1896) [n:127.0.0.1:35234_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 240144 INFO (qtp2132072037-1896) [n:127.0.0.1:35234_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.c.SyncStrategy http://127.0.0.1:35234/collection1_shard3_replica_n63/ has no replicas
[junit4] 2> 240144 INFO (qtp2132072037-1896) [n:127.0.0.1:35234_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 240151 INFO (qtp2132072037-1896) [n:127.0.0.1:35234_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:35234/collection1_shard3_replica_n63/ shard3
[junit4] 2> 240302 INFO (qtp2132072037-1896) [n:127.0.0.1:35234_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 240305 INFO (qtp2132072037-1896) [n:127.0.0.1:35234_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard3_replica_n63&action=CREATE&collection=collection1&shard=shard3&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1414
[junit4] 2> 240316 INFO (qtp2132072037-1891) [n:127.0.0.1:35234_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:35234_&action=ADDREPLICA&collection=collection1&shard=shard3&type=NRT&wt=javabin&version=2} status=0 QTime=1434
[junit4] 2> 240467 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_1D9FE4F5C38DA91E-001/shard-3-001 of type NRT
[junit4] 2> 240468 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T08:27:37+11:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 240471 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 240471 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 240471 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.e.j.s.session Scavenging every 600000ms
[junit4] 2> 240471 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2d88679d{/,null,AVAILABLE}
[junit4] 2> 240472 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@717db804{HTTP/1.1,[http/1.1]}{127.0.0.1:38962}
[junit4] 2> 240472 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.e.j.s.Server Started @243562ms
[junit4] 2> 240472 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_1D9FE4F5C38DA91E-001/tempDir-001/jetty3, solrconfig=solrconfig.xml, hostContext=/, hostPort=38962, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_1D9FE4F5C38DA91E-001/shard-3-001/cores}
[junit4] 2> 240475 ERROR (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 240475 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 8.0.0
[junit4] 2> 240475 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 240475 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 240475 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-03-10T05:58:17.221Z
[junit4] 2> 240481 INFO (zkConnectionManagerCallback-423-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 240486 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 240486 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_1D9FE4F5C38DA91E-001/shard-3-001/solr.xml
[junit4] 2> 240491 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 240491 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 240493 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@4c47c964, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 240499 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40476/solr
[junit4] 2> 240505 INFO (zkConnectionManagerCallback-427-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 240509 INFO (zkConnectionManagerCallback-429-thread-1-processing-n:127.0.0.1:38962_) [n:127.0.0.1:38962_ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 240534 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [n:127.0.0.1:38962_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 240543 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [n:127.0.0.1:38962_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 240547 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [n:127.0.0.1:38962_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38962_
[junit4] 2> 240554 INFO (zkCallback-400-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 240554 INFO (zkCallback-408-thread-1-processing-n:127.0.0.1:50070_) [n:127.0.0.1:50070_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 240555 INFO (zkCallback-418-thread-1-processing-n:127.0.0.1:35234_) [n:127.0.0.1:35234_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 240555 INFO (zkCallback-390-thread-1-processing-n:127.0.0.1:56858_) [n:127.0.0.1:56858_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 240560 INFO (zkCallback-428-thread-1-processing-n:127.0.0.1:38962_) [n:127.0.0.1:38962_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 240648 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [n:127.0.0.1:38962_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4c47c964
[junit4] 2> 240662 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [n:127.0.0.1:38962_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4c47c964
[junit4] 2> 240662 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [n:127.0.0.1:38962_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4c47c964
[junit4] 2> 240664 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [n:127.0.0.1:38962_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_1D9FE4F5C38DA91E-001/shard-3-001/cores
[junit4] 2> 240732 INFO (qtp2132072037-1894) [n:127.0.0.1:35234_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:38962_&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 240745 INFO (OverseerCollectionConfigSetProcessor-73428174100889604-127.0.0.1:56858_-n_0000000000) [n:127.0.0.1:56858_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000006 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 240747 INFO (OverseerThreadFactory-552-thread-5-processing-n:127.0.0.1:56858_) [n:127.0.0.1:56858_ ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:38962_ for creating new replica
[junit4] 2> 240761 INFO (qtp842432612-1922) [n:127.0.0.1:38962_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n65&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 240762 INFO (qtp842432612-1922) [n:127.0.0.1:38962_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 241792 INFO (qtp842432612-1922) [n:127.0.0.1:38962_ c:collection1 s:shard1 x:collection1_shard1_replica_n65] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 241807 INFO (qtp842432612-1922) [n:127.0.0.1:38962_ c:collection1 s:shard1 x:collection1_shard1_replica_n65] o.a.s.s.IndexSchema [collection1_shard1_replica_n65] Schema name=test
[junit4] 2> 241902 INFO (qtp842432612-1922) [n:127.0.0.1:38962_ c:collection1 s:shard1 x:collection1_shard1_replica_n65] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 241929 INFO (qtp842432612-1922) [n:127.0.0.1:38962_ c:collection1 s:shard1 x:collection1_shard1_replica_n65] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n65' using configuration from collection collection1, trusted=true
[junit4] 2> 241930 INFO (qtp842432612-1922) [n:127.0.0.1:38962_ c:collection1 s:shard1 x:collection1_shard1_replica_n65] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_n65' (registry 'solr.core.collection1.shard1.replica_n65') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4c47c964
[junit4] 2> 241930 INFO (qtp842432612-1922) [n:127.0.0.1:38962_ c:collection1 s:shard1 x:collection1_shard1_replica_n65] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 241930 INFO (qtp842432612-1922) [n:127.0.0.1:38962_ c:collection1 s:shard1 x:collection1_shard1_replica_n65] o.a.s.c.SolrCore [[collection1_shard1_replica_n65] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_1D9FE4F5C38DA91E-001/shard-3-001/cores/collection1_shard1_replica_n65], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_1D9FE4F5C38DA91E-001/shard-3-001/cores/collection1_shard1_replica_n65/data/]
[junit4] 2> 241933 INFO (qtp842432612-1922) [n:127.0.0.1:38962_ c:collection1 s:shard1 x:collection1_shard1_replica_n65] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=15, maxMergeAtOnceExplicit=30, maxMergedSegmentMB=59.888671875, floorSegmentMB=0.5185546875, forceMergeDeletesPctAllowed=18.31411628818963, segmentsPerTier=16.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.3948930594730886
[junit4] 2> 241937 WARN (qtp842432612-1922) [n:127.0.0.1:38962_ c:collection1 s:shard1 x:collection1_shard1_replica_n65] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 241996 INFO (qtp842432612-1922) [n:127.0.0.1:38962_ c:collection1 s:shard1 x:collection1_shard1_replica_n65] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 241996 INFO (qtp842432612-1922) [n:127.0.0.1:38962_ c:collection1 s:shard1 x:collection1_shard1_replica_n65] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 241998 INFO (qtp842432612-1922) [n:127.0.0.1:38962_ c:collection1 s:shard1 x:collection1_shard1_replica_n65] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 241998 INFO (qtp842432612-1922) [n:127.0.0.1:38962_ c:collection1 s:shard1 x:collection1_shard1_replica_n65] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 241999 INFO (qtp842432612-1922) [n:127.0.0.1:38962_ c:collection1 s:shard1 x:collection1_shard1_replica_n65] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=14, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 242000 INFO (qtp842432612-1922) [n:127.0.0.1:38962_ c:collection1 s:shard1 x:collection1_shard1_replica_n65] o.a.s.s.SolrIndexSearcher Opening [Searcher@7014ad66[collection1_shard1_replica_n65] main]
[junit4] 2> 242001 INFO (qtp842432612-1922) [n:127.0.0.1:38962_ c:collection1 s:shard1 x:collection1_shard1_replica_n65] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 242001 INFO (qtp842432612-1922) [n:127.0.0.1:38962_ c:collection1 s:shard1 x:collection1_shard1_replica_n65] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 242002 INFO (qtp842432612-1922) [n:127.0.0.1:38962_ c:collection1 s:shard1 x:collection1_shard1_replica_n65] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 242003 INFO (qtp842432612-1922) [n:127.0.0.1:38962_ c:collection1 s:shard1 x:collection1_shard1_replica_n65] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1594529151712231424
[junit4] 2> 242006 INFO (searcherExecutor-588-thread-1-processing-n:127.0.0.1:38962_ x:collection1_shard1_replica_n65 s:shard1 c:collection1) [n:127.0.0.1:38962_ c:collection1 s:shard1 x:collection1_shard1_replica_n65] o.a.s.c.SolrCore [collection1_shard1_replica_n65] Registered new searcher Searcher@7014ad66[collection1_shard1_replica_n65] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 242010 INFO (qtp842432612-1922) [n:127.0.0.1:38962_ c:collection1 s:shard1 x:collection1_shard1_replica_n65] o.a.s.c.ZkShardTerms Successful update terms at /collections/collection1/terms/shard1 to Terms{values={core_node66=0}, version=0}
[junit4] 2> 242015 INFO (qtp842432612-1922) [n:127.0.0.1:38962_ c:collection1 s:shard1 x:collection1_shard1_replica_n65] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 242015 INFO (qtp842432612-1922) [n:127.0.0.1:38962_ c:collection1 s:shard1 x:collection1_shard1_replica_n65] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 242015 INFO (qtp842432612-1922) [n:127.0.0.1:38962_ c:collection1 s:shard1 x:collection1_shard1_replica_n65] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:38962/collection1_shard1_replica_n65/
[junit4] 2> 242015 INFO (qtp842432612-1922) [n:127.0.0.1:38962_ c:collection1 s:shard1 x:collection1_shard1_replica_n65] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 242015 INFO (qtp842432612-1922) [n:127.0.0.1:38962_ c:collection1 s:shard1 x:collection1_shard1_replica_n65] o.a.s.c.SyncStrategy http://127.0.0.1:38962/collection1_shard1_replica_n65/ has no replicas
[junit4] 2> 242015 INFO (qtp842432612-1922) [n:127.0.0.1:38962_ c:collection1 s:shard1 x:collection1_shard1_replica_n65] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 242019 INFO (qtp842432612-1922) [n:127.0.0.1:38962_ c:collection1 s:shard1 x:collection1_shard1_replica_n65] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:38962/collection1_shard1_replica_n65/ shard1
[junit4] 2> 242170 INFO (qtp842432612-1922) [n:127.0.0.1:38962_ c:collection1 s:shard1 x:collection1_shard1_replica_n65] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 242178 INFO (qtp842432612-1922) [n:127.0.0.1:38962_ c:collection1 s:shard1 x:collection1_shard1_replica_n65] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n65&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1416
[junit4] 2> 242183 INFO (qtp2132072037-1894) [n:127.0.0.1:35234_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:38962_&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2} status=0 QTime=1450
[junit4] 2> 242397 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 4 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_1D9FE4F5C38DA91E-001/shard-4-001 of type NRT
[junit4] 2> 242398 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T08:27:37+11:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 242413 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 242414 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 242414 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.e.j.s.session Scavenging every 660000ms
[junit4] 2> 242414 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3fc1e7bd{/,null,AVAILABLE}
[junit4] 2> 242415 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@2cfe19b6{HTTP/1.1,[http/1.1]}{127.0.0.1:54616}
[junit4] 2> 242415 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.e.j.s.Server Started @245505ms
[junit4] 2> 242415 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_1D9FE4F5C38DA91E-001/tempDir-001/jetty4, solrconfig=solrconfig.xml, hostContext=/, hostPort=54616, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_1D9FE4F5C38DA91E-001/shard-4-001/cores}
[junit4] 2> 242415 ERROR (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 242416 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 8.0.0
[junit4] 2> 242416 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 242416 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 242416 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-03-10T05:58:19.162Z
[junit4] 2> 242431 INFO (zkConnectionManagerCallback-433-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 242433 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 242433 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_1D9FE4F5C38DA91E-001/shard-4-001/solr.xml
[junit4] 2> 242437 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 242438 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 242457 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@4c47c964, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 242460 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40476/solr
[junit4] 2> 242463 INFO (zkConnectionManagerCallback-437-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 242466 INFO (zkConnectionManagerCallback-439-thread-1-processing-n:127.0.0.1:54616_) [n:127.0.0.1:54616_ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 242474 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [n:127.0.0.1:54616_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 242478 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [n:127.0.0.1:54616_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 242480 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [n:127.0.0.1:54616_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54616_
[junit4] 2> 242482 INFO (zkCallback-418-thread-1-processing-n:127.0.0.1:35234_) [n:127.0.0.1:35234_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 242482 INFO (zkCallback-390-thread-2-processing-n:127.0.0.1:56858_) [n:127.0.0.1:56858_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 242483 INFO (zkCallback-408-thread-1-processing-n:127.0.0.1:50070_) [n:127.0.0.1:50070_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 242483 INFO (zkCallback-400-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 242483 INFO (zkCallback-428-thread-1-processing-n:127.0.0.1:38962_) [n:127.0.0.1:38962_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 242498 INFO (zkCallback-438-thread-1-processing-n:127.0.0.1:54616_) [n:127.0.0.1:54616_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 242589 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [n:127.0.0.1:54616_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4c47c964
[junit4] 2> 242602 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [n:127.0.0.1:54616_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4c47c964
[junit4] 2> 242602 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [n:127.0.0.1:54616_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4c47c964
[junit4] 2> 242604 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [n:127.0.0.1:54616_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_1D9FE4F5C38DA91E-001/shard-4-001/cores
[junit4] 2> 242673 INFO (qtp2132072037-1896) [n:127.0.0.1:35234_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:54616_&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 242677 INFO (OverseerCollectionConfigSetProcessor-73428174100889604-127.0.0.1:56858_-n_0000000000) [n:127.0.0.1:56858_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000008 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 242677 INFO (OverseerThreadFactory-552-thread-5-processing-n:127.0.0.1:56858_) [n:127.0.0.1:56858_ ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:54616_ for creating
[...truncated too long message...]
_ c:collection2 s:shard3 r:core_node15 x:collection2_shard3_replica_n11] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection2.shard3.replica_n11, tag=1882014118
[junit4] 2> 3856768 INFO (zkCallback-449-thread-14-processing-n:127.0.0.1:59718_) [n:127.0.0.1:59718_ c:collection2 s:shard3 r:core_node15 x:collection2_shard3_replica_n11] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@4419bb2f: rootName = null, domain = solr.core.collection2.shard3.replica_n11, service url = null, agent id = null] for registry solr.core.collection2.shard3.replica_n11 / com.codahale.metrics.MetricRegistry@1a38344a
[junit4] 2> 3856783 INFO (zkCallback-449-thread-14-processing-n:127.0.0.1:59718_) [n:127.0.0.1:59718_ c:collection2 s:shard3 r:core_node15 x:collection2_shard3_replica_n11] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.collection2.shard3.leader, tag=1882014118
[junit4] 2> 3856789 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@413292b8{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 3856789 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@33f9b9b5{/,null,UNAVAILABLE}
[junit4] 2> 3856789 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.e.j.s.session Stopped scavenging
[junit4] 2> 3856790 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.ChaosMonkey monkey: stop jetty! 53511
[junit4] 2> 3856790 ERROR (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 3856792 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[1D9FE4F5C38DA91E]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:40476 40476
[junit4] 2> 3856894 INFO (Thread-457) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:40476 40476
[junit4] 2> 3856895 WARN (Thread-457) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 18 /solr/collections/collection2/terms/shard3
[junit4] 2> 18 /solr/aliases.json
[junit4] 2> 14 /solr/configs/conf1
[junit4] 2> 14 /solr/collections/collection1/terms/shard2
[junit4] 2> 14 /solr/collections/collection1/terms/shard3
[junit4] 2> 13 /solr/collections/collection2/terms/shard7
[junit4] 2> 13 /solr/collections/collection2/terms/shard5
[junit4] 2> 13 /solr/collections/collection2/terms/shard6
[junit4] 2> 13 /solr/collections/collection2/terms/shard4
[junit4] 2> 13 /solr/collections/collection2/terms/shard1
[junit4] 2> 13 /solr/collections/collection2/terms/shard2
[junit4] 2> 11 /solr/collections/collection1/terms/shard1
[junit4] 2> 10 /solr/security.json
[junit4] 2> 8 /solr/collections/collection2/state.json
[junit4] 2> 6 /solr/collections/collection1/collectionprops.json
[junit4] 2> 6 /solr/collections/collection1/state.json
[junit4] 2> 5 /solr/collections/collection2/collectionprops.json
[junit4] 2> 3 /solr/collections/control_collection/terms/shard1
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 22 /solr/clusterprops.json
[junit4] 2> 18 /solr/clusterstate.json
[junit4] 2> 3 /solr/autoscaling.json
[junit4] 2> 2 /solr/overseer_elect/election/73428174100889612-127.0.0.1:35234_-n_0000000002
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 21 /solr/live_nodes
[junit4] 2> 18 /solr/collections
[junit4] 2> 3 /solr/overseer/queue
[junit4] 2> 3 /solr/autoscaling/events/.scheduled_maintenance
[junit4] 2> 3 /solr/overseer/collection-queue-work
[junit4] 2> 3 /solr/overseer/queue-work
[junit4] 2>
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=FullSolrCloudDistribCmdsTest -Dtests.method=test -Dtests.seed=1D9FE4F5C38DA91E -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=fr-CH -Dtests.timezone=Asia/Ulan_Bator -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
[junit4] ERROR 3624s J0 | FullSolrCloudDistribCmdsTest.test <<<
[junit4] > Throwable #1: java.lang.OutOfMemoryError: Java heap space
[junit4] > at __randomizedtesting.SeedInfo.seed([1D9FE4F5C38DA91E:95CBDB2F6D71C4E6]:0)
[junit4] > at java.util.Arrays.copyOf(Arrays.java:3332)
[junit4] > at java.lang.AbstractStringBuilder.ensureCapacityInternal(AbstractStringBuilder.java:124)
[junit4] > at java.lang.AbstractStringBuilder.append(AbstractStringBuilder.java:649)
[junit4] > at java.lang.StringBuilder.append(StringBuilder.java:202)
[junit4] > at org.apache.http.client.utils.URLEncodedUtils.urlEncode(URLEncodedUtils.java:536)
[junit4] > at org.apache.http.client.utils.URLEncodedUtils.encodeFormFields(URLEncodedUtils.java:652)
[junit4] > at org.apache.http.client.utils.URLEncodedUtils.format(URLEncodedUtils.java:404)
[junit4] > at org.apache.http.client.utils.URLEncodedUtils.format(URLEncodedUtils.java:382)
[junit4] > at org.apache.http.client.entity.UrlEncodedFormEntity.<init>(UrlEncodedFormEntity.java:75)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.fillContentStream(HttpSolrClient.java:513)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:420)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
[junit4] > at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
[junit4] > at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
[junit4] > at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1104)
[junit4] > at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:884)
[junit4] > at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:817)
[junit4] > at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
[junit4] > at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:974)
[junit4] > at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:990)
[junit4] > at org.apache.solr.cloud.CloudInspectUtil.compareResults(CloudInspectUtil.java:228)
[junit4] > at org.apache.solr.cloud.CloudInspectUtil.compareResults(CloudInspectUtil.java:167)
[junit4] > at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.testIndexingBatchPerRequestWithHttpSolrClient(FullSolrCloudDistribCmdsTest.java:668)
[junit4] > at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test(FullSolrCloudDistribCmdsTest.java:152)
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_1D9FE4F5C38DA91E-001
[junit4] 2> Mar 10, 2018 6:58:33 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Lucene70, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@59274e0b), locale=fr-CH, timezone=Asia/Ulan_Bator
[junit4] 2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=94579056,total=461373440
[junit4] 2> NOTE: All tests run in this JVM: [HdfsRestartWhileUpdatingTest, TestFieldCacheReopen, SuggestComponentContextFilterQueryTest, MergeStrategyTest, TestMiniSolrCloudClusterSSL, TestLuceneMatchVersion, TestFreeTextSuggestions, DefaultValueUpdateProcessorTest, WrapperMergePolicyFactoryTest, TestTolerantUpdateProcessorCloud, FullSolrCloudDistribCmdsTest]
[junit4] Completed [446/784 (1!)] on J0 in 3624.98s, 1 test, 1 error <<< FAILURES!
[...truncated 1238 lines...]
[junit4] JVM J0: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J0-20180310_055413_5973295966706284717301.sysout
[junit4] >>> JVM J0 emitted unexpected output (verbatim) ----
[junit4] java.lang.OutOfMemoryError: Java heap space
[junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps/java_pid24411.hprof ...
[junit4] Heap dump file created [572936267 bytes in 17.216 secs]
[junit4] <<< JVM J0: EOF ----
[...truncated 9273 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:636: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:570: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid24411.hprof
Total time: 344 minutes 25 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1498 - Still
unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1498/
2 tests failed.
FAILED: org.apache.solr.cloud.MoveReplicaHDFSTest.testFailedMove
Error Message:
Stack Trace:
java.lang.AssertionError
at __randomizedtesting.SeedInfo.seed([3A2FC374AAA74EC3:90E210861D749B13]:0)
at org.junit.Assert.fail(Assert.java:92)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertFalse(Assert.java:68)
at org.junit.Assert.assertFalse(Assert.java:79)
at org.apache.solr.cloud.MoveReplicaTest.testFailedMove(MoveReplicaTest.java:303)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.MoveReplicaHDFSTest.testNormalMove
Error Message:
Error from server at https://127.0.0.1:33155/solr: delete the collection time out:180s
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:33155/solr: delete the collection time out:180s
at __randomizedtesting.SeedInfo.seed([3A2FC374AAA74EC3:9CF774D32F468CD9]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1104)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:884)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:817)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
at org.apache.solr.cloud.MiniSolrCloudCluster.deleteAllCollections(MiniSolrCloudCluster.java:451)
at org.apache.solr.cloud.MoveReplicaTest.beforeTest(MoveReplicaTest.java:97)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:968)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 14547 lines...]
[junit4] Suite: org.apache.solr.cloud.MoveReplicaHDFSTest
[junit4] 2> 2500756 INFO (SUITE-MoveReplicaHDFSTest-seed#[3A2FC374AAA74EC3]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.MoveReplicaHDFSTest_3A2FC374AAA74EC3-001/init-core-data-001
[junit4] 2> 2500757 WARN (SUITE-MoveReplicaHDFSTest-seed#[3A2FC374AAA74EC3]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=256 numCloses=256
[junit4] 2> 2500758 INFO (SUITE-MoveReplicaHDFSTest-seed#[3A2FC374AAA74EC3]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 2500759 INFO (SUITE-MoveReplicaHDFSTest-seed#[3A2FC374AAA74EC3]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 2500761 INFO (SUITE-MoveReplicaHDFSTest-seed#[3A2FC374AAA74EC3]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 4 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.MoveReplicaHDFSTest_3A2FC374AAA74EC3-001/tempDir-001
[junit4] 2> 2500761 INFO (SUITE-MoveReplicaHDFSTest-seed#[3A2FC374AAA74EC3]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 2500763 INFO (Thread-14031) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 2500765 INFO (Thread-14031) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 2500773 ERROR (Thread-14031) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 2500864 INFO (SUITE-MoveReplicaHDFSTest-seed#[3A2FC374AAA74EC3]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:38690
[junit4] 2> 2500874 INFO (zkConnectionManagerCallback-3868-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2500885 INFO (jetty-launcher-3865-thread-1) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T15:27:37-06:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 2500887 INFO (jetty-launcher-3865-thread-4) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T15:27:37-06:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 2500888 INFO (jetty-launcher-3865-thread-3) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T15:27:37-06:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 2500889 INFO (jetty-launcher-3865-thread-2) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T15:27:37-06:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 2500889 INFO (jetty-launcher-3865-thread-1) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 2500889 INFO (jetty-launcher-3865-thread-1) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 2500889 INFO (jetty-launcher-3865-thread-1) [ ] o.e.j.s.session Scavenging every 600000ms
[junit4] 2> 2500890 INFO (jetty-launcher-3865-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6b00d9cb{/solr,null,AVAILABLE}
[junit4] 2> 2500893 INFO (jetty-launcher-3865-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@17bbb107{SSL,[ssl, http/1.1]}{127.0.0.1:38989}
[junit4] 2> 2500893 INFO (jetty-launcher-3865-thread-1) [ ] o.e.j.s.Server Started @2503986ms
[junit4] 2> 2500893 INFO (jetty-launcher-3865-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=38989}
[junit4] 2> 2500893 ERROR (jetty-launcher-3865-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2500894 INFO (jetty-launcher-3865-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.0.0
[junit4] 2> 2500894 INFO (jetty-launcher-3865-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2500894 INFO (jetty-launcher-3865-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2500894 INFO (jetty-launcher-3865-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-03-09T08:52:57.308Z
[junit4] 2> 2500895 INFO (jetty-launcher-3865-thread-2) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 2500895 INFO (jetty-launcher-3865-thread-2) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 2500895 INFO (jetty-launcher-3865-thread-2) [ ] o.e.j.s.session Scavenging every 600000ms
[junit4] 2> 2500896 INFO (jetty-launcher-3865-thread-3) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 2500896 INFO (jetty-launcher-3865-thread-3) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 2500896 INFO (jetty-launcher-3865-thread-3) [ ] o.e.j.s.session Scavenging every 660000ms
[junit4] 2> 2500896 INFO (jetty-launcher-3865-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@475a367d{/solr,null,AVAILABLE}
[junit4] 2> 2500896 INFO (jetty-launcher-3865-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@2b778467{SSL,[ssl, http/1.1]}{127.0.0.1:33155}
[junit4] 2> 2500897 INFO (jetty-launcher-3865-thread-2) [ ] o.e.j.s.Server Started @2503989ms
[junit4] 2> 2500897 INFO (jetty-launcher-3865-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=33155}
[junit4] 2> 2500897 ERROR (jetty-launcher-3865-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2500897 INFO (jetty-launcher-3865-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.0.0
[junit4] 2> 2500897 INFO (jetty-launcher-3865-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2500897 INFO (jetty-launcher-3865-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2500897 INFO (jetty-launcher-3865-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-03-09T08:52:57.311Z
[junit4] 2> 2500898 INFO (zkConnectionManagerCallback-3870-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2500898 INFO (jetty-launcher-3865-thread-3) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@44e8108c{/solr,null,AVAILABLE}
[junit4] 2> 2500899 INFO (jetty-launcher-3865-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2500899 INFO (jetty-launcher-3865-thread-3) [ ] o.e.j.s.AbstractConnector Started ServerConnector@1ae7366e{SSL,[ssl, http/1.1]}{127.0.0.1:60838}
[junit4] 2> 2500899 INFO (jetty-launcher-3865-thread-3) [ ] o.e.j.s.Server Started @2503992ms
[junit4] 2> 2500899 INFO (jetty-launcher-3865-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=60838}
[junit4] 2> 2500899 ERROR (jetty-launcher-3865-thread-3) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2500899 INFO (jetty-launcher-3865-thread-3) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.0.0
[junit4] 2> 2500899 INFO (jetty-launcher-3865-thread-3) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2500900 INFO (jetty-launcher-3865-thread-3) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2500900 INFO (jetty-launcher-3865-thread-3) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-03-09T08:52:57.314Z
[junit4] 2> 2500901 INFO (zkConnectionManagerCallback-3872-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2500901 INFO (jetty-launcher-3865-thread-4) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 2500901 INFO (jetty-launcher-3865-thread-4) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 2500901 INFO (jetty-launcher-3865-thread-4) [ ] o.e.j.s.session Scavenging every 600000ms
[junit4] 2> 2500901 INFO (zkConnectionManagerCallback-3874-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2500902 INFO (jetty-launcher-3865-thread-4) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4955d81d{/solr,null,AVAILABLE}
[junit4] 2> 2500902 INFO (jetty-launcher-3865-thread-4) [ ] o.e.j.s.AbstractConnector Started ServerConnector@40ebf93b{SSL,[ssl, http/1.1]}{127.0.0.1:50839}
[junit4] 2> 2500902 INFO (jetty-launcher-3865-thread-4) [ ] o.e.j.s.Server Started @2503995ms
[junit4] 2> 2500902 INFO (jetty-launcher-3865-thread-4) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=50839}
[junit4] 2> 2500902 ERROR (jetty-launcher-3865-thread-4) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2500902 INFO (jetty-launcher-3865-thread-4) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.0.0
[junit4] 2> 2500902 INFO (jetty-launcher-3865-thread-4) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2500903 INFO (jetty-launcher-3865-thread-4) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2500903 INFO (jetty-launcher-3865-thread-4) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-03-09T08:52:57.317Z
[junit4] 2> 2500905 INFO (jetty-launcher-3865-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2500906 INFO (jetty-launcher-3865-thread-3) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2500906 INFO (zkConnectionManagerCallback-3876-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2500911 INFO (jetty-launcher-3865-thread-4) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2500913 INFO (jetty-launcher-3865-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38690/solr
[junit4] 2> 2500919 INFO (jetty-launcher-3865-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38690/solr
[junit4] 2> 2500921 INFO (jetty-launcher-3865-thread-3) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38690/solr
[junit4] 2> 2500929 INFO (zkConnectionManagerCallback-3880-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2500930 INFO (jetty-launcher-3865-thread-4) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38690/solr
[junit4] 2> 2500930 INFO (zkConnectionManagerCallback-3888-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2500933 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x104da02b5630005, likely client has closed socket
[junit4] 2> 2500933 INFO (zkConnectionManagerCallback-3892-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2500933 DEBUG (jetty-launcher-3865-thread-1) [n:127.0.0.1:38989_solr ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$283/552291203@5f089314
[junit4] 2> 2500933 INFO (zkConnectionManagerCallback-3884-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2500936 DEBUG (jetty-launcher-3865-thread-3) [n:127.0.0.1:60838_solr ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$283/552291203@576ddd35
[junit4] 2> 2500937 DEBUG (jetty-launcher-3865-thread-4) [n:127.0.0.1:50839_solr ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$283/552291203@713e8f8b
[junit4] 2> 2500936 INFO (zkConnectionManagerCallback-3894-thread-1-processing-n:127.0.0.1:38989_solr) [n:127.0.0.1:38989_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2500938 DEBUG (jetty-launcher-3865-thread-2) [n:127.0.0.1:33155_solr ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$283/552291203@68db1aec
[junit4] 2> 2500939 INFO (zkConnectionManagerCallback-3896-thread-1-processing-n:127.0.0.1:60838_solr) [n:127.0.0.1:60838_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2500939 INFO (zkConnectionManagerCallback-3898-thread-1-processing-n:127.0.0.1:50839_solr) [n:127.0.0.1:50839_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2500940 INFO (zkConnectionManagerCallback-3900-thread-1-processing-n:127.0.0.1:33155_solr) [n:127.0.0.1:33155_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2500976 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterprops.json
[junit4] 2> 2500976 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 2500977 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterstate.json
[junit4] 2> 2500977 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 2500978 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/aliases.json
[junit4] 2> 2500978 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/security.json
[junit4] 2> 2500983 INFO (jetty-launcher-3865-thread-4) [n:127.0.0.1:50839_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2500983 DEBUG (jetty-launcher-3865-thread-4) [n:127.0.0.1:50839_solr ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/73423199112724491-127.0.0.1:50839_solr-n_0000000000
[junit4] 2> 2500983 INFO (jetty-launcher-3865-thread-4) [n:127.0.0.1:50839_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:50839_solr
[junit4] 2> 2500984 INFO (jetty-launcher-3865-thread-4) [n:127.0.0.1:50839_solr ] o.a.s.c.Overseer Overseer (id=73423199112724491-127.0.0.1:50839_solr-n_0000000000) starting
[junit4] 2> 2500995 DEBUG (OverseerStateUpdate-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.Overseer Starting to work on the main queue
[junit4] 2> 2500995 DEBUG (OverseerCollectionConfigSetProcessor-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.OverseerTaskProcessor Process current queue of overseer operations
[junit4] 2> 2500996 INFO (jetty-launcher-3865-thread-4) [n:127.0.0.1:50839_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50839_solr
[junit4] 2> 2500997 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 2500997 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue-work
[junit4] 2> 2501003 DEBUG (OverseerAutoScalingTriggerThread-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
[junit4] 2> 2501005 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
[junit4] 2> 2501005 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 2501005 INFO (zkCallback-3897-thread-1-processing-n:127.0.0.1:50839_solr) [n:127.0.0.1:50839_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2501006 DEBUG (OverseerCollectionConfigSetProcessor-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
[junit4] 2> 2501006 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/autoscaling.json
[junit4] 2> 2501006 DEBUG (OverseerCollectionConfigSetProcessor-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
[junit4] 2> 2501006 DEBUG (OverseerCollectionConfigSetProcessor-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
[junit4] 2> 2501006 DEBUG (OverseerCollectionConfigSetProcessor-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
[junit4] 2> 2501006 DEBUG (OverseerCollectionConfigSetProcessor-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
[junit4] 2> 2501007 DEBUG (OverseerCollectionConfigSetProcessor-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
[junit4] 2> 2501007 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/collection-queue-work
[junit4] 2> 2501008 DEBUG (OverseerAutoScalingTriggerThread-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
[junit4] 2> 2501011 DEBUG (OverseerAutoScalingTriggerThread-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:50839_solr]
[junit4] 2> 2501011 DEBUG (OverseerAutoScalingTriggerThread-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion -1
[junit4] 2> 2501011 DEBUG (OverseerAutoScalingTriggerThread-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
[junit4] 2> 2501016 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/autoscaling/events/.scheduled_maintenance
[junit4] 2> 2501018 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/autoscaling/events/.auto_add_replicas
[junit4] 2> 2501019 DEBUG (OverseerAutoScalingTriggerThread-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 2501019 DEBUG (ScheduledTrigger-9277-thread-1) [n:127.0.0.1:50839_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 2501019 DEBUG (OverseerAutoScalingTriggerThread-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
[junit4] 2> 2501104 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterprops.json
[junit4] 2> 2501104 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 2501104 INFO (jetty-launcher-3865-thread-1) [n:127.0.0.1:38989_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2501104 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterstate.json
[junit4] 2> 2501105 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 2501105 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/aliases.json
[junit4] 2> 2501106 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/security.json
[junit4] 2> 2501108 INFO (jetty-launcher-3865-thread-1) [n:127.0.0.1:38989_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2501108 DEBUG (jetty-launcher-3865-thread-1) [n:127.0.0.1:38989_solr ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/73423199112724489-127.0.0.1:38989_solr-n_0000000001
[junit4] 2> 2501109 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/overseer_elect/election/73423199112724491-127.0.0.1:50839_solr-n_0000000000
[junit4] 2> 2501109 DEBUG (jetty-launcher-3865-thread-1) [n:127.0.0.1:38989_solr ] o.a.s.c.LeaderElector Watching path /overseer_elect/election/73423199112724491-127.0.0.1:50839_solr-n_0000000000 to know if I could be the leader
[junit4] 2> 2501110 DEBUG (jetty-launcher-3865-thread-1) [n:127.0.0.1:38989_solr ] o.a.s.c.ZkController Publish node=127.0.0.1:38989_solr as DOWN
[junit4] 2> 2501111 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/queue
[junit4] 2> 2501112 INFO (jetty-launcher-3865-thread-1) [n:127.0.0.1:38989_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38989_solr
[junit4] 2> 2501112 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
[junit4] 2> 2501112 DEBUG (OverseerStateUpdate-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"downnode",
[junit4] 2> "node_name":"127.0.0.1:38989_solr"} current state version: 0
[junit4] 2> 2501112 DEBUG (OverseerStateUpdate-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.o.NodeMutator DownNode state invoked for node: 127.0.0.1:38989_solr
[junit4] 2> 2501114 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 2501114 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 2501114 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 2501115 INFO (zkCallback-3897-thread-1-processing-n:127.0.0.1:50839_solr) [n:127.0.0.1:50839_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2501116 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterprops.json
[junit4] 2> 2501116 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 2501116 INFO (jetty-launcher-3865-thread-3) [n:127.0.0.1:60838_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 2501116 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterstate.json
[junit4] 2> 2501117 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 2501117 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/aliases.json
[junit4] 2> 2501118 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/security.json
[junit4] 2> 2501119 INFO (jetty-launcher-3865-thread-3) [n:127.0.0.1:60838_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2501119 DEBUG (jetty-launcher-3865-thread-3) [n:127.0.0.1:60838_solr ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/73423199112724490-127.0.0.1:60838_solr-n_0000000002
[junit4] 2> 2501120 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/overseer_elect/election/73423199112724489-127.0.0.1:38989_solr-n_0000000001
[junit4] 2> 2501120 DEBUG (jetty-launcher-3865-thread-3) [n:127.0.0.1:60838_solr ] o.a.s.c.LeaderElector Watching path /overseer_elect/election/73423199112724489-127.0.0.1:38989_solr-n_0000000001 to know if I could be the leader
[junit4] 2> 2501120 DEBUG (jetty-launcher-3865-thread-3) [n:127.0.0.1:60838_solr ] o.a.s.c.ZkController Publish node=127.0.0.1:60838_solr as DOWN
[junit4] 2> 2501121 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/queue
[junit4] 2> 2501122 INFO (jetty-launcher-3865-thread-3) [n:127.0.0.1:60838_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60838_solr
[junit4] 2> 2501122 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
[junit4] 2> 2501122 DEBUG (OverseerStateUpdate-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.Overseer processMessage: queueSize: 2, message = {
[junit4] 2> "operation":"downnode",
[junit4] 2> "node_name":"127.0.0.1:60838_solr"} current state version: 0
[junit4] 2> 2501122 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 2501122 DEBUG (OverseerStateUpdate-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.o.NodeMutator DownNode state invoked for node: 127.0.0.1:60838_solr
[junit4] 2> 2501123 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 2501123 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 2501129 INFO (zkCallback-3897-thread-1-processing-n:127.0.0.1:50839_solr) [n:127.0.0.1:50839_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2501129 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 2501130 INFO (zkCallback-3893-thread-1-processing-n:127.0.0.1:38989_solr) [n:127.0.0.1:38989_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (3)
[junit4] 2> 2501130 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 2501130 INFO (zkCallback-3895-thread-1-processing-n:127.0.0.1:60838_solr) [n:127.0.0.1:60838_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2501131 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterprops.json
[junit4] 2> 2501132 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 2501132 INFO (jetty-launcher-3865-thread-2) [n:127.0.0.1:33155_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 2501132 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterstate.json
[junit4] 2> 2501132 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 2501133 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/aliases.json
[junit4] 2> 2501133 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/security.json
[junit4] 2> 2501134 INFO (jetty-launcher-3865-thread-2) [n:127.0.0.1:33155_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2501135 DEBUG (jetty-launcher-3865-thread-2) [n:127.0.0.1:33155_solr ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/73423199112724492-127.0.0.1:33155_solr-n_0000000003
[junit4] 2> 2501135 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/overseer_elect/election/73423199112724490-127.0.0.1:60838_solr-n_0000000002
[junit4] 2> 2501135 DEBUG (jetty-launcher-3865-thread-2) [n:127.0.0.1:33155_solr ] o.a.s.c.LeaderElector Watching path /overseer_elect/election/73423199112724490-127.0.0.1:60838_solr-n_0000000002 to know if I could be the leader
[junit4] 2> 2501135 DEBUG (jetty-launcher-3865-thread-2) [n:127.0.0.1:33155_solr ] o.a.s.c.ZkController Publish node=127.0.0.1:33155_solr as DOWN
[junit4] 2> 2501136 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/queue
[junit4] 2> 2501137 INFO (jetty-launcher-3865-thread-2) [n:127.0.0.1:33155_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33155_solr
[junit4] 2> 2501137 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
[junit4] 2> 2501137 DEBUG (OverseerStateUpdate-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.Overseer processMessage: queueSize: 3, message = {
[junit4] 2> "operation":"downnode",
[junit4] 2> "node_name":"127.0.0.1:33155_solr"} current state version: 0
[junit4] 2> 2501137 DEBUG (OverseerStateUpdate-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.o.NodeMutator DownNode state invoked for node: 127.0.0.1:33155_solr
[junit4] 2> 2501137 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 2501137 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 2501138 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 2501138 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 2501138 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 2501138 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 2501138 INFO (zkCallback-3895-thread-1-processing-n:127.0.0.1:60838_solr) [n:127.0.0.1:60838_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2501140 INFO (zkCallback-3897-thread-1-processing-n:127.0.0.1:50839_solr) [n:127.0.0.1:50839_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2501142 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 2501143 INFO (zkCallback-3893-thread-1-processing-n:127.0.0.1:38989_solr) [n:127.0.0.1:38989_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2501143 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 2501144 INFO (zkCallback-3899-thread-1-processing-n:127.0.0.1:33155_solr) [n:127.0.0.1:33155_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2501228 INFO (jetty-launcher-3865-thread-1) [n:127.0.0.1:38989_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38989.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4f995404
[junit4] 2> 2501238 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/queue
[junit4] 2> 2501239 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
[junit4] 2> 2501240 INFO (jetty-launcher-3865-thread-3) [n:127.0.0.1:60838_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60838.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4f995404
[junit4] 2> 2501255 INFO (jetty-launcher-3865-thread-4) [n:127.0.0.1:50839_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50839.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4f995404
[junit4] 2> 2501255 INFO (jetty-launcher-3865-thread-2) [n:127.0.0.1:33155_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33155.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4f995404
[junit4] 2> 2501287 INFO (jetty-launcher-3865-thread-1) [n:127.0.0.1:38989_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38989.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4f995404
[junit4] 2> 2501287 INFO (jetty-launcher-3865-thread-1) [n:127.0.0.1:38989_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38989.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4f995404
[junit4] 2> 2501289 INFO (jetty-launcher-3865-thread-3) [n:127.0.0.1:60838_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60838.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4f995404
[junit4] 2> 2501290 INFO (jetty-launcher-3865-thread-3) [n:127.0.0.1:60838_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60838.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4f995404
[junit4] 2> 2501292 INFO (jetty-launcher-3865-thread-1) [n:127.0.0.1:38989_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.MoveReplicaHDFSTest_3A2FC374AAA74EC3-001/tempDir-001/node1/.
[junit4] 2> 2501293 INFO (jetty-launcher-3865-thread-3) [n:127.0.0.1:60838_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.MoveReplicaHDFSTest_3A2FC374AAA74EC3-001/tempDir-001/node3/.
[junit4] 2> 2501297 INFO (jetty-launcher-3865-thread-4) [n:127.0.0.1:50839_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50839.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4f995404
[junit4] 2> 2501297 INFO (jetty-launcher-3865-thread-4) [n:127.0.0.1:50839_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50839.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4f995404
[junit4] 2> 2501299 INFO (jetty-launcher-3865-thread-4) [n:127.0.0.1:50839_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.MoveReplicaHDFSTest_3A2FC374AAA74EC3-001/tempDir-001/node2/.
[junit4] 2> 2501301 INFO (jetty-launcher-3865-thread-2) [n:127.0.0.1:33155_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33155.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4f995404
[junit4] 2> 2501301 INFO (jetty-launcher-3865-thread-2) [n:127.0.0.1:33155_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33155.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4f995404
[junit4] 2> 2501303 INFO (jetty-launcher-3865-thread-2) [n:127.0.0.1:33155_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.MoveReplicaHDFSTest_3A2FC374AAA74EC3-001/tempDir-001/node4/.
[junit4] 2> 2501511 INFO (zkConnectionManagerCallback-3910-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2501516 INFO (zkConnectionManagerCallback-3914-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2501516 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterprops.json
[junit4] 2> 2501517 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 2501517 INFO (SUITE-MoveReplicaHDFSTest-seed#[3A2FC374AAA74EC3]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 2501517 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterstate.json
[junit4] 2> 2501518 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 2501518 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/aliases.json
[junit4] 2> 2501518 INFO (SUITE-MoveReplicaHDFSTest-seed#[3A2FC374AAA74EC3]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:38690/solr ready
[junit4] 2> 2501556 INFO (qtp819545665-29656) [n:127.0.0.1:60838_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :overseerstatus with params action=OVERSEERSTATUS&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2501556 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/collection-queue-work
[junit4] 2> 2501557 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/collection-queue-work
[junit4] 2> 2501557 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/overseer/collection-queue-work/qnr-0000000000
[junit4] 2> 2501557 DEBUG (OverseerCollectionConfigSetProcessor-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.ZkDistributedQueue Found child node with improper name: qnr-0000000000
[junit4] 2> 2501557 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/collection-queue-work
[junit4] 2> 2501558 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/collection-queue-work
[junit4] 2> 2501558 DEBUG (OverseerCollectionConfigSetProcessor-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.ZkDistributedQueue Found child node with improper name: qnr-0000000000
[junit4] 2> 2501558 DEBUG (OverseerCollectionConfigSetProcessor-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.OverseerTaskQueue Returning topN elements: [/overseer/collection-queue-work/qn-0000000000, ]
[junit4] 2> 2501559 DEBUG (OverseerCollectionConfigSetProcessor-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.OverseerTaskProcessor Got 1 tasks from work-queue : [[org.apache.solr.cloud.OverseerTaskQueue$QueueEvent@2e684ecf]]
[junit4] 2> 2501559 DEBUG (OverseerCollectionConfigSetProcessor-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.OverseerTaskProcessor Marked task [/overseer/collection-queue-work/qn-0000000000] as running
[junit4] 2> 2501559 DEBUG (OverseerCollectionConfigSetProcessor-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Get the message id:/overseer/collection-queue-work/qn-0000000000 message:{"operation":"overseerstatus"}
[junit4] 2> 2501559 DEBUG (OverseerThreadFactory-9279-thread-1-processing-n:127.0.0.1:50839_solr) [n:127.0.0.1:50839_solr ] o.a.s.c.OverseerTaskProcessor Runner processing /overseer/collection-queue-work/qn-0000000000
[junit4] 2> 2501559 DEBUG (OverseerThreadFactory-9279-thread-1-processing-n:127.0.0.1:50839_solr) [n:127.0.0.1:50839_solr ] o.a.s.c.a.c.OverseerCollectionMessageHandler OverseerCollectionMessageHandler.processMessage : overseerstatus , {"operation":"overseerstatus"}
[junit4] 2> 2501559 DEBUG (OverseerCollectionConfigSetProcessor-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 1
[junit4] 2> 2501559 DEBUG (OverseerCollectionConfigSetProcessor-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.OverseerTaskProcessor RunningTasks: [/overseer/collection-queue-work/qn-0000000000]
[junit4] 2> 2501559 DEBUG (OverseerCollectionConfigSetProcessor-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
[junit4] 2> 2501559 DEBUG (OverseerCollectionConfigSetProcessor-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
[junit4] 2> 2501559 DEBUG (OverseerCollectionConfigSetProcessor-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: [/overseer/collection-queue-work/qn-0000000000]
[junit4] 2> 2501560 DEBUG (OverseerCollectionConfigSetProcessor-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.OverseerTaskQueue Peeking for top 99 elements. ExcludeSet: [/overseer/collection-queue-work/qn-0000000000],[]
[junit4] 2> 2501562 DEBUG (OverseerThreadFactory-9279-thread-1-processing-n:127.0.0.1:50839_solr) [n:127.0.0.1:50839_solr ] o.a.s.c.OverseerTaskProcessor Completed task:[/overseer/collection-queue-work/qn-0000000000]
[junit4] 2> 2501562 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: data: /solr/overseer/collection-queue-work/qnr-0000000000
[junit4] 2> 2501562 DEBUG (zkCallback-3895-thread-1-processing-n:127.0.0.1:60838_solr) [n:127.0.0.1:60838_solr ] o.a.s.c.OverseerTaskQueue NodeDataChanged fired on path /overseer/collection-queue-work/qnr-0000000000 state SyncConnected latchEventType null
[junit4] 2> 2501563 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/collection-queue-work
[junit4] 2> 2501563 DEBUG (OverseerThreadFactory-9279-thread-1-processing-n:127.0.0.1:50839_solr) [n:127.0.0.1:50839_solr ] o.a.s.c.OverseerTaskProcessor Marked task [/overseer/collection-queue-work/qn-0000000000] as completed.
[junit4] 2> 2501563 DEBUG (OverseerThreadFactory-9279-thread-1-processing-n:127.0.0.1:50839_solr) [n:127.0.0.1:50839_solr ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
[junit4] 2> 2501563 DEBUG (OverseerThreadFactory-9279-thread-1-processing-n:127.0.0.1:50839_solr) [n:127.0.0.1:50839_solr ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
[junit4] 2> 2501563 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/collection-queue-work
[junit4] 2> 2501563 DEBUG (OverseerThreadFactory-9279-thread-1-processing-n:127.0.0.1:50839_solr) [n:127.0.0.1:50839_solr ] o.a.s.c.OverseerTaskProcessor CompletedTasks: [/overseer/collection-queue-work/qn-0000000000]
[junit4] 2> 2501563 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/collection-queue-work
[junit4] 2> 2501563 DEBUG (OverseerThreadFactory-9279-thread-1-processing-n:127.0.0.1:50839_solr) [n:127.0.0.1:50839_solr ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: [/overseer/collection-queue-work/qn-0000000000]
[junit4] 2> 2501564 DEBUG (OverseerThreadFactory-9279-thread-1-processing-n:127.0.0.1:50839_solr) [n:127.0.0.1:50839_solr ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Message id:/overseer/collection-queue-work/qn-0000000000 complete, response:{leader=127.0.0.1:50839_solr,overseer_queue_size=0,overseer_work_queue_size=0,overseer_collection_queue_size=2,overseer_operations={am_i_leader={requests=3,errors=0,avgRequestsPerSecond=5.306252400183781,5minRateRequestsPerSecond=0.0,15minRateRequestsPerSecond=0.0,avgTimePerRequest=0.3592053333333333,medianRequestTime=0.34719,75thPcRequestTime=0.43594,95thPcRequestTime=0.43594,99thPcRequestTime=0.43594,999thPcRequestTime=0.43594},downnode={requests=3,errors=0,avgRequestsPerSecond=6.697837060780126,5minRateRequestsPerSecond=0.0,15minRateRequestsPerSecond=0.0,avgTimePerRequest=0.6004856666666666,medianRequestTime=0.133451,75thPcRequestTime=1.609773,95thPcRequestTime=1.609773,99thPcRequestTime=1.609773,999thPcRequestTime=1.609773}},collection_operations={am_i_leader={requests=3,errors=0,avgRequestsPerSecond=5.312210003135798,5minRateRequestsPerSecond=0.0,15minRateRequestsPerSecond=0.0,avgTimePerRequest=0.356649,medianRequestTime=0.282434,75thPcRequestTime=0.57368,95thPcRequestTime=0.57368,99thPcRequestTime=0.57368,999thPcRequestTime=0.57368},overseerstatus={requests=0,errors=0,avgRequestsPerSecond=0.0,5minRateRequestsPerSecond=0.0,15minRateRequestsPerSecond=0.0,avgTimePerRequest=0.0,medianRequestTime=0.0,75thPcRequestTime=0.0,95thPcRequestTime=0.0,99thPcRequestTime=0.0,999thPcRequestTime=0.0}},overseer_queue={},overseer_internal_queue={peek={avgRequestsPerSecond=1.773678065189242,5minRateRequestsPerSecond=0.0,15minRateRequestsPerSecond=0.0,avgTimePerRequest=3.964448,medianRequestTime=3.964448,75thPcRequestTime=3.964448,95thPcRequestTime=3.964448,99thPcRequestTime=3.964448,999thPcRequestTime=3.964448}},collection_queue={peektopn_wait2000={avgRequestsPerSecond=1.8065391724769393,5minRateRequestsPerSecond=0.0,15minRateRequestsPerSecond=0.0,avgTimePerRequest=551.689304,medianRequestTime=551.689304,75thPcRequestTime=551.689304,95thPcRequestTime=551.689304,99thPcRequestTime=551.689304,999thPcRequestTime=551.689304}}}
[junit4] 2> 2501564 DEBUG (OverseerCollectionConfigSetProcessor-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.ZkDistributedQueue Found child node with improper name: qnr-0000000000
[junit4] 2> 2501564 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/collection-queue-work
[junit4] 2> 2501564 INFO (qtp819545665-29656) [n:127.0.0.1:60838_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=OVERSEERSTATUS&wt=javabin&version=2} status=0 QTime=8
[junit4] 1> Formatting using clusterid: testClusterID
[junit4] 2> 2501610 WARN (SUITE-MoveReplicaHDFSTest-seed#[3A2FC374AAA74EC3]-worker) [ ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
[junit4] 2> 2501623 WARN (SUITE-MoveReplicaHDFSTest-seed#[3A2FC374AAA74EC3]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 2501626 INFO (SUITE-MoveReplicaHDFSTest-seed#[3A2FC374AAA74EC3]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 2501643 INFO (SUITE-MoveReplicaHDFSTest-seed#[3A2FC374AAA74EC3]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/hdfs to ./temp/Jetty_localhost_44995_hdfs____x3rl8f/webapp
[junit4] 2> 2502021 DEBUG (ScheduledTrigger-9277-thread-3) [n:127.0.0.1:50839_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
[junit4] 2> 2502032 INFO (SUITE-MoveReplicaHDFSTest-seed#[3A2FC374AAA74EC3]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:44995
[junit4] 2> 2502133 WARN (SUITE-MoveReplicaHDFSTest-seed#[3A2FC374AAA74EC3]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 2502134 INFO (SUITE-MoveReplicaHDFSTest-seed#[3A2FC374AAA74EC3]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 2502149 INFO (SUITE-MoveReplicaHDFSTest-seed#[3A2FC374AAA74EC3]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_33553_datanode____xq6aoq/webapp
[junit4] 2> 2502657 INFO (SUITE-MoveReplicaHDFSTest-seed#[3A2FC374AAA74EC3]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:33553
[junit4] 2> 2502764 WARN (SUITE-MoveReplicaHDFSTest-seed#[3A2FC374AAA74EC3]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 2502765 INFO (SUITE-MoveReplicaHDFSTest-seed#[3A2FC374AAA74EC3]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 2502787 INFO (SUITE-MoveReplicaHDFSTest-seed#[3A2FC374AAA74EC3]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_57745_datanode____m7f4sd/webapp
[junit4] 2> 2502847 ERROR (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.MoveReplicaHDFSTest_3A2FC374AAA74EC3-001/tempDir-002/hdfsBaseDir/data/data1/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.MoveReplicaHDFSTest_3A2FC374AAA74EC3-001/tempDir-002/hdfsBaseDir/data/data2/]] heartbeating to localhost/127.0.0.1:46295) [ ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
[junit4] 2> 2502868 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x4a079451544de0: from storage DS-c62ff0df-60eb-42b5-9124-af460782bf84 node DatanodeRegistration(127.0.0.1:52349, datanodeUuid=1be33c35-a854-4af9-8654-11eeeb3b88c3, infoPort=52370, infoSecurePort=0, ipcPort=36837, storageInfo=lv=-56;cid=testClusterID;nsid=763785840;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
[junit4] 2> 2502868 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x4a079451544de0: from storage DS-907c305e-51cb-45a5-92e3-118b08c100fd node DatanodeRegistration(127.0.0.1:52349, datanodeUuid=1be33c35-a854-4af9-8654-11eeeb3b88c3, infoPort=52370, infoSecurePort=0, ipcPort=36837, storageInfo=lv=-56;cid=testClusterID;nsid=763785840;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 2503022 DEBUG (ScheduledTrigger-9277-thread-4) [n:127.0.0.1:50839_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
[junit4] 2> 2503286 INFO (SUITE-MoveReplicaHDFSTest-seed#[3A2FC374AAA74EC3]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:57745
[junit4] 2> 2503509 ERROR (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.MoveReplicaHDFSTest_3A2FC374AAA74EC3-001/tempDir-002/hdfsBaseDir/data/data3/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.MoveReplicaHDFSTest_3A2FC374AAA74EC3-001/tempDir-002/hdfsBaseDir/data/data4/]] heartbeating to localhost/127.0.0.1:46295) [ ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
[junit4] 2> 2503517 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x4a0794781c0154: from storage DS-a65462db-a184-494b-9f98-67a85fb70df9 node DatanodeRegistration(127.0.0.1:53782, datanodeUuid=60f15929-a6fe-419b-9764-aee630156c21, infoPort=49680, infoSecurePort=0, ipcPort=37935, storageInfo=lv=-56;cid=testClusterID;nsid=763785840;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
[junit4] 2> 2503518 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x4a0794781c0154: from storage DS-1106e61c-5254-4a32-b3d3-d6c12263ab92 node DatanodeRegistration(127.0.0.1:53782, datanodeUuid=60f15929-a6fe-419b-9764-aee630156c21, infoPort=49680, infoSecurePort=0, ipcPort=37935, storageInfo=lv=-56;cid=testClusterID;nsid=763785840;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 2503561 DEBUG (OverseerCollectionConfigSetProcessor-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
[junit4] 2> 2503562 DEBUG (OverseerCollectionConfigSetProcessor-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
[junit4] 2> 2503562 INFO (OverseerCollectionConfigSetProcessor-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 2503562 DEBUG (OverseerCollectionConfigSetProcessor-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
[junit4] 2> 2503562 DEBUG (OverseerCollectionConfigSetProcessor-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
[junit4] 2> 2503562 DEBUG (OverseerCollectionConfigSetProcessor-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
[junit4] 2> 2503562 DEBUG (OverseerCollectionConfigSetProcessor-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
[junit4] 2> 2503563 DEBUG (OverseerCollectionConfigSetProcessor-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
[junit4] IGNOR/A 0.00s J1 | MoveReplicaHDFSTest.testNormalFailedMove
[junit4] > Assumption #1: 'badapple' test group is disabled (@BadApple(bugUrl=https://issues.apache.org/jira/browse/SOLR-12028))
[junit4] 2> 2503833 INFO (TEST-MoveReplicaHDFSTest.test-seed#[3A2FC374AAA74EC3]) [ ] o.a.s.SolrTestCaseJ4 ###Starting test
[junit4] 2> 2503834 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterprops.json
[junit4] 2> 2503834 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 2503835 INFO (TEST-MoveReplicaHDFSTest.test-seed#[3A2FC374AAA74EC3]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 2503835 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterstate.json
[junit4] 2> 2503835 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 2503835 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/aliases.json
[junit4] 2> 2503837 INFO (TEST-MoveReplicaHDFSTest.test-seed#[3A2FC374AAA74EC3]) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T15:27:37-06:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 2503865 INFO (TEST-MoveReplicaHDFSTest.test-seed#[3A2FC374AAA74EC3]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 2503865 INFO (TEST-MoveReplicaHDFSTest.test-seed#[3A2FC374AAA74EC3]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 2503865 INFO (TEST-MoveReplicaHDFSTest.test-seed#[3A2FC374AAA74EC3]) [ ] o.e.j.s.session Scavenging every 600000ms
[junit4] 2> 2503868 INFO (TEST-MoveReplicaHDFSTest.test-seed#[3A2FC374AAA74EC3]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1351cd2d{/solr,null,AVAILABLE}
[junit4] 2> 2503868 INFO (TEST-MoveReplicaHDFSTest.test-seed#[3A2FC374AAA74EC3]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@cb1b24e{SSL,[ssl, http/1.1]}{127.0.0.1:57722}
[junit4] 2> 2503868 INFO (TEST-MoveReplicaHDFSTest.test-seed#[3A2FC374AAA74EC3]) [ ] o.e.j.s.Server Started @2506961ms
[junit4] 2> 2503869 INFO (TEST-MoveReplicaHDFSTest.test-seed#[3A2FC374AAA74EC3]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=57722}
[junit4] 2> 2503869 ERROR (TEST-MoveReplicaHDFSTest.test-seed#[3A2FC374AAA74EC3]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2503869 INFO (TEST-MoveReplicaHDFSTest.test-seed#[3A2FC374AAA74EC3]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.0.0
[junit4] 2> 2503869 INFO (TEST-MoveReplicaHDFSTest.test-seed#[3A2FC374AAA74EC3]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2503869 INFO (TEST-MoveReplicaHDFSTest.test-seed#[3A2FC374AAA74EC3]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2503869 INFO (TEST-MoveReplicaHDFSTest.test-seed#[3A2FC374AAA74EC3]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-03-09T08:53:00.283Z
[junit4] 2> 2503871 INFO (zkConnectionManagerCallback-3917-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2503872 INFO (TEST-MoveReplicaHDFSTest.test-seed#[3A2FC374AAA74EC3]) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2503882 INFO (TEST-MoveReplicaHDFSTest.test-seed#[3A2FC374AAA74EC3]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38690/solr
[junit4] 2> 2503884 INFO (zkConnectionManagerCallback-3921-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2503885 DEBUG (TEST-MoveReplicaHDFSTest.test-seed#[3A2FC374AAA74EC3]) [n:127.0.0.1:57722_solr ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$283/552291203@7a742842
[junit4] 2> 2503886 INFO (zkConnectionManagerCallback-3923-thread-1-processing-n:127.0.0.1:57722_solr) [n:127.0.0.1:57722_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2503892 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterprops.json
[junit4] 2> 2503893 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 2503893 INFO (TEST-MoveReplicaHDFSTest.test-seed#[3A2FC374AAA74EC3]) [n:127.0.0.1:57722_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 2503893 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterstate.json
[junit4] 2> 2503893 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 2503893 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/aliases.json
[junit4] 2> 2503894 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/security.json
[junit4] 2> 2503895 INFO (TEST-MoveReplicaHDFSTest.test-seed#[3A2FC374AAA74EC3]) [n:127.0.0.1:57722_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2503895 DEBUG (TEST-MoveReplicaHDFSTest.test-seed#[3A2FC374AAA74EC3]) [n:127.0.0.1:57722_solr ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/73423199112724497-127.0.0.1:57722_solr-n_0000000004
[junit4] 2> 2503896 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/overseer_elect/election/73423199112724492-127.0.0.1:33155_solr-n_0000000003
[junit4] 2> 2503896 DEBUG (TEST-MoveReplicaHDFSTest.test-seed#[3A2FC374AAA74EC3]) [n:127.0.0.1:57722_solr ] o.a.s.c.LeaderElector Watching path /overseer_elect/election/73423199112724492-127.0.0.1:33155_solr-n_0000000003 to know if I could be the leader
[junit4] 2> 2503896 DEBUG (TEST-MoveReplicaHDFSTest.test-seed#[3A2FC374AAA74EC3]) [n:127.0.0.1:57722_solr ] o.a.s.c.ZkController Publish node=127.0.0.1:57722_solr as DOWN
[junit4] 2> 2503897 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/queue
[junit4] 2> 2503898 INFO (TEST-MoveReplicaHDFSTest.test-seed#[3A2FC374AAA74EC3]) [n:127.0.0.1:57722_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57722_solr
[junit4] 2> 2503898 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
[junit4] 2> 2503898 DEBUG (OverseerStateUpdate-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"downnode",
[junit4] 2> "node_name":"127.0.0.1:57722_solr"} current state version: 0
[junit4] 2> 2503898 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 2503898 DEBUG (OverseerStateUpdate-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.o.NodeMutator DownNode state invoked for node: 127.0.0.1:57722_solr
[junit4] 2> 2503898 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 2503899 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 2503899 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 2503899 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 2503899 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 2503900 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 2503900 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 2503900 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 2503900 INFO (zkCallback-3893-thread-1-processing-n:127.0.0.1:38989_solr) [n:127.0.0.1:38989_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 2503900 INFO (zkCallback-3895-thread-1-processing-n:127.0.0.1:60838_solr) [n:127.0.0.1:60838_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 2503900 INFO (zkCallback-3897-thread-1-processing-n:127.0.0.1:50839_solr) [n:127.0.0.1:50839_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 2503901 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 2503901 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 2503901 INFO (zkCallback-3899-thread-1-processing-n:127.0.0.1:33155_solr) [n:127.0.0.1:33155_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 2503901 INFO (zkCallback-3913-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 2503902 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 2503902 INFO (zkCallback-3913-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 2503909 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 2503909 INFO (zkCallback-3922-thread-1-processing-n:127.0.0.1:57722_solr) [n:127.0.0.1:57722_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 2503999 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/queue
[junit4] 2> 2504000 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
[junit4] 2> 2504022 DEBUG (ScheduledTrigger-9277-thread-3) [n:127.0.0.1:50839_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 5
[junit4] 2> 2504071 INFO (TEST-MoveReplicaHDFSTest.test-seed#[3A2FC374AAA74EC3]) [n:127.0.0.1:57722_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_57722.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4f995404
[junit4] 2> 2504087 INFO (TEST-MoveReplicaHDFSTest.test-seed#[3A2FC374AAA74EC3]) [n:127.0.0.1:57722_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_57722.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4f995404
[junit4] 2> 2504087 INFO (TEST-MoveReplicaHDFSTest.test-seed#[3A2FC374AAA74EC3]) [n:127.0.0.1:57722_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_57722.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4f995404
[junit4] 2> 2504089 INFO (TEST-MoveReplicaHDFSTest.test-seed#[3A2FC374AAA74EC3]) [n:127.0.0.1:57722_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.MoveReplicaHDFSTest_3A2FC374AAA74EC3-001/tempDir-001/node5/.
[junit4] 2> 2504177 INFO (zkConnectionManagerCallback-3927-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2504179 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x104da02b5630012, likely client has closed socket
[junit4] 2> 2504179 INFO (TEST-MoveReplicaHDFSTest.test-seed#[3A2FC374AAA74EC3]) [ ] o.a.s.c.MoveReplicaTest total_jettys: 5
[junit4] 2> 2504213 INFO (qtp1558530514-29649) [n:127.0.0.1:33155_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=2&collection.configName=conf1&maxShardsPerNode=2&autoAddReplicas=false&name=MoveReplicaHDFSTest_coll_true&nrtReplicas=2&action=CREATE&numShards=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2504217 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/collection-queue-work
[junit4] 2> 2504217 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/overseer/collection-queue-work/qnr-0000000002
[junit4] 2> 2504217 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/collection-queue-work
[junit4] 2> 2504217 DEBUG (OverseerCollectionConfigSetProcessor-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.ZkDistributedQueue Found child node with improper name: qnr-0000000002
[junit4] 2> 2504218 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/collection-queue-work
[junit4] 2> 2504218 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/collection-queue-work
[junit4] 2> 2504218 DEBUG (OverseerCollectionConfigSetProcessor-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.ZkDistributedQueue Found child node with improper name: qnr-0000000002
[junit4] 2> 2504218 DEBUG (OverseerCollectionConfigSetProcessor-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.OverseerTaskQueue Returning topN elements: [/overseer/collection-queue-work/qn-0000000002, ]
[junit4] 2> 2504219 DEBUG (OverseerCollectionConfigSetProcessor-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.OverseerTaskProcessor Got 1 tasks from work-queue : [[org.apache.solr.cloud.OverseerTaskQueue$QueueEvent@2e684ed1]]
[junit4] 2> 2504219 DEBUG (OverseerCollectionConfigSetProcessor-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.OverseerTaskProcessor Marked task [/overseer/collection-queue-work/qn-0000000002] as running
[junit4] 2> 2504219 DEBUG (OverseerCollectionConfigSetProcessor-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Get the message id:/overseer/collection-queue-work/qn-0000000002 message:{
[junit4] 2> "name":"MoveReplicaHDFSTest_coll_true",
[junit4] 2> "fromApi":"true",
[junit4] 2> "replicationFactor":"2",
[junit4] 2> "collection.configName":"conf1",
[junit4] 2> "numShards":"2",
[junit4] 2> "maxShardsPerNode":"2",
[junit4] 2> "autoAddReplicas":"false",
[junit4] 2> "nrtReplicas":"2",
[junit4] 2> "stateFormat":"2",
[junit4] 2> "operation":"create"}
[junit4] 2> 2504219 DEBUG (OverseerThreadFactory-9279-thread-2-processing-n:127.0.0.1:50839_solr) [n:127.0.0.1:50839_solr ] o.a.s.c.OverseerTaskProcessor Runner processing /overseer/collection-queue-work/qn-0000000002
[junit4] 2> 2504219 DEBUG (OverseerThreadFactory-9279-thread-2-processing-n:127.0.0.1:50839_solr) [n:127.0.0.1:50839_solr ] o.a.s.c.a.c.OverseerCollectionMessageHandler OverseerCollectionMessageHandler.processMessage : create , {
[junit4] 2> "name":"MoveReplicaHDFSTest_coll_true",
[junit4] 2> "fromApi":"true",
[junit4] 2> "replicationFactor":"2",
[junit4] 2> "collection.configName":"conf1",
[junit4] 2> "numShards":"2",
[junit4] 2> "maxShardsPerNode":"2",
[junit4] 2> "autoAddReplicas":"false",
[junit4] 2> "nrtReplicas":"2",
[junit4] 2> "stateFormat":"2",
[junit4] 2> "operation":"create"}
[junit4] 2> 2504219 DEBUG (OverseerCollectionConfigSetProcessor-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 1
[junit4] 2> 2504220 DEBUG (OverseerCollectionConfigSetProcessor-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.OverseerTaskProcessor RunningTasks: [/overseer/collection-queue-work/qn-0000000002]
[junit4] 2> 2504220 DEBUG (OverseerCollectionConfigSetProcessor-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
[junit4] 2> 2504220 DEBUG (OverseerCollectionConfigSetProcessor-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
[junit4] 2> 2504220 DEBUG (OverseerCollectionConfigSetProcessor-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: [/overseer/collection-queue-work/qn-0000000002]
[junit4] 2> 2504219 INFO (OverseerThreadFactory-9279-thread-2-processing-n:127.0.0.1:50839_solr) [n:127.0.0.1:50839_solr ] o.a.s.c.a.c.CreateCollectionCmd Create collection MoveReplicaHDFSTest_coll_true
[junit4] 2> 2504220 DEBUG (OverseerCollectionConfigSetProcessor-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.OverseerTaskQueue Peeking for top 99 elements. ExcludeSet: [/overseer/collection-queue-work/qn-0000000002],[]
[junit4] 2> 2504221 DEBUG (OverseerThreadFactory-9279-thread-2-processing-n:127.0.0.1:50839_solr) [n:127.0.0.1:50839_solr ] o.a.s.c.a.c.Assign Identify nodes using default
[junit4] 2> 2504221 DEBUG (OverseerThreadFactory-9279-thread-2-processing-n:127.0.0.1:50839_solr) [n:127.0.0.1:50839_solr ] o.a.s.c.a.c.OverseerCollectionMessageHandler creating collections conf node /collections/MoveReplicaHDFSTest_coll_true
[junit4] 2> 2504223 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/collections
[junit4] 2> 2504223 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/collections
[junit4] 2> 2504223 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/collections
[junit4] 2> 2504223 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/collections
[junit4] 2> 2504223 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/collections
[junit4] 2> 2504223 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/collections
[junit4] 2> 2504224 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 2504224 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 2504224 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 2504224 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 2504224 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 2504224 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 2504224 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 2504225 DEBUG (OverseerThreadFactory-9279-thread-2-processing-n:127.0.0.1:50839_solr) [n:127.0.0.1:50839_solr ] o.a.s.c.a.c.CreateCollectionCmd Check for collection zkNode:MoveReplicaHDFSTest_coll_true
[junit4] 2> 2504226 DEBUG (OverseerThreadFactory-9279-thread-2-processing-n:127.0.0.1:50839_solr) [n:127.0.0.1:50839_solr ] o.a.s.c.a.c.CreateCollectionCmd Collection zkNode exists
[junit4] 2> 2504228 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/queue
[junit4] 2> 2504229 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
[junit4] 2> 2504229 DEBUG (OverseerStateUpdate-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "name":"MoveReplicaHDFSTest_coll_true",
[junit4] 2> "fromApi":"true",
[junit4] 2> "replicationFactor":"2",
[junit4] 2> "collection.configName":"conf1",
[junit4] 2> "numShards":"2",
[junit4] 2> "maxShardsPerNode":"2",
[junit4] 2> "autoAddReplicas":"false",
[junit4] 2> "nrtReplicas":"2",
[junit4] 2> "stateFormat":"2",
[junit4] 2> "operation":"create"} current state version: 0
[junit4] 2> 2504229 DEBUG (OverseerStateUpdate-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.o.ClusterStateMutator building a new cName: MoveReplicaHDFSTest_coll_true
[junit4] 2> 2504230 DEBUG (OverseerStateUpdate-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/MoveReplicaHDFSTest_coll_true/state.json
[junit4] 2> 2504233 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/queue
[junit4] 2> 2504234 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
[junit4] 2> 2504329 DEBUG (OverseerThreadFactory-9279-thread-2-processing-n:127.0.0.1:50839_solr) [n:127.0.0.1:50839_solr ] o.a.s.c.a.c.CreateCollectionCmd Creating SolrCores for new collection MoveReplicaHDFSTest_coll_true, shardNames [shard1, shard2] , message : {
[junit4] 2> "name":"MoveReplicaHDFSTest_coll_true",
[junit4] 2> "fromApi":"true",
[junit4] 2> "replicationFactor":"2",
[junit4] 2> "collection.configName":"conf1",
[junit4] 2> "numShards":"2",
[junit4] 2> "maxShardsPerNode":"2",
[junit4] 2> "autoAddReplicas":"false",
[junit4] 2> "nrtReplicas":"2",
[junit4] 2> "stateFormat":"2",
[junit4] 2> "operation":"create"}
[junit4] 2> 2504331 DEBUG (OverseerThreadFactory-9279-thread-2-processing-n:127.0.0.1:50839_solr) [n:127.0.0.1:50839_solr ] o.a.s.c.a.c.CreateCollectionCmd Creating core MoveReplicaHDFSTest_coll_true_shard1_replica_n1 as part of shard shard1 of collection MoveReplicaHDFSTest_coll_true on 127.0.0.1:57722_solr
[junit4] 2> 2504332 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/queue
[junit4] 2> 2504332 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
[junit4] 2> 2504333 DEBUG (OverseerStateUpdate-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"MoveReplicaHDFSTest_coll_true",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"MoveReplicaHDFSTest_coll_true_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:57722/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"} current state version: 0
[junit4] 2> 2504333 INFO (OverseerStateUpdate-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"MoveReplicaHDFSTest_coll_true",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"MoveReplicaHDFSTest_coll_true_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:57722/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 2504334 DEBUG (OverseerThreadFactory-9279-thread-2-processing-n:127.0.0.1:50839_solr) [n:127.0.0.1:50839_solr ] o.a.s.c.a.c.CreateCollectionCmd Creating core MoveReplicaHDFSTest_coll_true_shard1_replica_n2 as part of shard shard1 of collection MoveReplicaHDFSTest_coll_true on 127.0.0.1:50839_solr
[junit4] 2> 2504335 DEBUG (OverseerStateUpdate-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.o.SliceMutator Old Slice: shard1:{
[junit4] 2> "range":"80000000-ffffffff",
[junit4] 2> "state":"active",
[junit4] 2> "replicas":{}}
[junit4] 2> 2504335 DEBUG (OverseerStateUpdate-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.o.SliceMutator New Slice: shard1:{
[junit4] 2> "range":"80000000-ffffffff",
[junit4] 2> "state":"active",
[junit4] 2> "replicas":{"core_node3":{
[junit4] 2> "core":"MoveReplicaHDFSTest_coll_true_shard1_replica_n1",
[junit4] 2> "base_url":"https://127.0.0.1:57722/solr",
[junit4] 2> "state":"down",
[junit4] 2> "node_name":null,
[junit4] 2> "type":"NRT"}}}
[junit4] 2> 2504335 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/queue
[junit4] 2> 2504336 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
[junit4] 2> 2504336 DEBUG (OverseerStateUpdate-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.Overseer processMessage: queueSize: 2, message = {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"MoveReplicaHDFSTest_coll_true",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"MoveReplicaHDFSTest_coll_true_shard1_replica_n2",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:50839/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"} current state version: 0
[junit4] 2> 2504336 INFO (OverseerStateUpdate-73423199112724491-127.0.0.1:50839_solr-n_0000000000) [n:127.0.0.1:50839_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operati
[...truncated too long message...]
[junit4] 2> 9 /solr/clusterprops.json
[junit4] 2> 9 /solr/clusterstate.json
[junit4] 2> 3 /solr/overseer_elect/election/73423199112724491-127.0.0.1:50839_solr-n_0000000000
[junit4] 2> 2 /solr/overseer_elect/election/73423199112724489-127.0.0.1:38989_solr-n_0000000001
[junit4] 2> 2 /solr/collections/MoveReplicaHDFSTest_coll_true/leader_elect/shard2/election/73423199112724489-core_node7-n_0000000001
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 14 /solr/collections
[junit4] 2> 12 /solr/live_nodes
[junit4] 2>
[junit4] 2> 2829050 WARN (SUITE-MoveReplicaHDFSTest-seed#[3A2FC374AAA74EC3]-worker) [ ] o.a.h.h.s.d.DirectoryScanner DirectoryScanner: shutdown has been called
[junit4] 2> 2829081 INFO (SUITE-MoveReplicaHDFSTest-seed#[3A2FC374AAA74EC3]-worker) [ ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0
[junit4] 2> 2829183 WARN (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.MoveReplicaHDFSTest_3A2FC374AAA74EC3-001/tempDir-002/hdfsBaseDir/data/data3/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.MoveReplicaHDFSTest_3A2FC374AAA74EC3-001/tempDir-002/hdfsBaseDir/data/data4/]] heartbeating to localhost/127.0.0.1:46295) [ ] o.a.h.h.s.d.IncrementalBlockReportManager IncrementalBlockReportManager interrupted
[junit4] 2> 2829184 WARN (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.MoveReplicaHDFSTest_3A2FC374AAA74EC3-001/tempDir-002/hdfsBaseDir/data/data3/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.MoveReplicaHDFSTest_3A2FC374AAA74EC3-001/tempDir-002/hdfsBaseDir/data/data4/]] heartbeating to localhost/127.0.0.1:46295) [ ] o.a.h.h.s.d.DataNode Ending block pool service for: Block pool BP-1368281937-10.41.0.5-1520585577994 (Datanode Uuid 60f15929-a6fe-419b-9764-aee630156c21) service to localhost/127.0.0.1:46295
[junit4] 2> 2829191 WARN (SUITE-MoveReplicaHDFSTest-seed#[3A2FC374AAA74EC3]-worker) [ ] o.a.h.h.s.d.DirectoryScanner DirectoryScanner: shutdown has been called
[junit4] 2> 2829227 INFO (SUITE-MoveReplicaHDFSTest-seed#[3A2FC374AAA74EC3]-worker) [ ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0
[junit4] 2> 2829228 WARN (1205219577@qtp-1089428801-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:33553) [ ] o.a.h.h.HttpServer2 HttpServer Acceptor: isRunning is false. Rechecking.
[junit4] 2> 2829228 WARN (1205219577@qtp-1089428801-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:33553) [ ] o.a.h.h.HttpServer2 HttpServer Acceptor: isRunning is false
[junit4] 2> 2829232 WARN (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.MoveReplicaHDFSTest_3A2FC374AAA74EC3-001/tempDir-002/hdfsBaseDir/data/data1/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.MoveReplicaHDFSTest_3A2FC374AAA74EC3-001/tempDir-002/hdfsBaseDir/data/data2/]] heartbeating to localhost/127.0.0.1:46295) [ ] o.a.h.h.s.d.IncrementalBlockReportManager IncrementalBlockReportManager interrupted
[junit4] 2> 2829233 WARN (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.MoveReplicaHDFSTest_3A2FC374AAA74EC3-001/tempDir-002/hdfsBaseDir/data/data1/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.MoveReplicaHDFSTest_3A2FC374AAA74EC3-001/tempDir-002/hdfsBaseDir/data/data2/]] heartbeating to localhost/127.0.0.1:46295) [ ] o.a.h.h.s.d.DataNode Ending block pool service for: Block pool BP-1368281937-10.41.0.5-1520585577994 (Datanode Uuid 1be33c35-a854-4af9-8654-11eeeb3b88c3) service to localhost/127.0.0.1:46295
[junit4] 2> 2829249 INFO (SUITE-MoveReplicaHDFSTest-seed#[3A2FC374AAA74EC3]-worker) [ ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0
[junit4] 2> 2829383 INFO (SUITE-MoveReplicaHDFSTest-seed#[3A2FC374AAA74EC3]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:38690 38690
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.MoveReplicaHDFSTest_3A2FC374AAA74EC3-001
[junit4] 2> Mar 09, 2018 8:58:26 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 34 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Lucene70, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@1d7a7ad4), locale=lt-LT, timezone=Africa/Mbabane
[junit4] 2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=2,free=237261352,total=507510784
[junit4] 2> NOTE: All tests run in this JVM: [DistributedMLTComponentTest, TestConfigOverlay, LargeFieldTest, LeaderFailoverAfterPartitionTest, TestRawTransformer, SortSpecParsingTest, TestDistributedStatsComponentCardinality, DocExpirationUpdateProcessorFactoryTest, TestFieldSortValues, StatsReloadRaceTest, TestSweetSpotSimilarityFactory, SyncSliceTest, PrimUtilsTest, TestStressRecovery, AutoscalingHistoryHandlerTest, SolrJmxReporterTest, TestComplexPhraseLeadingWildcard, TestRecovery, TestLeaderElectionWithEmptyReplica, JsonLoaderTest, TestSolr4Spatial, TestCharFilters, SuggesterFSTTest, MetricUtilsTest, TestFieldCache, TestSchemaVersionResource, HdfsDirectoryFactoryTest, ForceLeaderTest, IndexSchemaTest, BitVectorTest, DistributedFacetPivotSmallTest, JvmMetricsTest, VMParamsZkACLAndCredentialsProvidersTest, MultiThreadedOCPTest, SpatialHeatmapFacetsTest, PeerSyncReplicationTest, SoftAutoCommitTest, TestSolrCoreSnapshots, BlockCacheTest, RuleEngineTest, TestTriggerIntegration, ResourceLoaderTest, SolrCoreTest, DistributedQueueTest, TestCollationFieldDocValues, TestReversedWildcardFilterFactory, CloudExitableDirectoryReaderTest, HttpSolrCallGetCoreTest, DistributedIntervalFacetingTest, UpdateLogTest, TestTrieFacet, CollectionsAPISolrJTest, TestCryptoKeys, TestImpersonationWithHadoopAuth, LoggingHandlerTest, PropertiesRequestHandlerTest, AutoScalingHandlerTest, TestMaxScoreQueryParser, HdfsRecoveryZkTest, TestPolicyCloud, RestartWhileUpdatingTest, TestUtilizeNode, QueryEqualityTest, TestElisionMultitermQuery, TestJavabinTupleStreamParser, TestCoreBackup, TestStressVersions, TestLRUCache, TestHashQParserPlugin, TestPerFieldSimilarityWithDefaultOverride, ConnectionManagerTest, ReplaceNodeNoTargetTest, EnumFieldTest, AlternateDirectoryTest, TestSchemaManager, TestSQLHandlerNonCloud, TestSolrJ, SliceStateTest, TestCloudConsistency, TestReplicationHandler, TestReloadDeadlock, TestSolrFieldCacheBean, TestUninvertingReader, SchemaVersionSpecificBehaviorTest, SchemaApiFailureTest, CdcrUpdateLogTest, CustomCollectionTest, TestSimpleTrackingShardHandler, TestStressCloudBlindAtomicUpdates, TestUseDocValuesAsStored, SimplePostToolTest, CurrencyFieldTypeTest, TestMacroExpander, BJQParserTest, UninvertDocValuesMergePolicyTest, HdfsThreadLeakTest, SpellCheckCollatorWithCollapseTest, TestCursorMarkWithoutUniqueKey, PeerSyncTest, TestComputePlanAction, TestNumericRangeQuery64, TestExportWriter, TestZkChroot, SolrGangliaReporterTest, TestDistributedGrouping, TestCloudRecovery, CdcrRequestHandlerTest, TestBM25SimilarityFactory, TestSystemCollAutoCreate, HdfsAutoAddReplicasIntegrationTest, TestHdfsUpdateLog, TestSolrConfigHandler, HighlighterMaxOffsetTest, DistributedQueryComponentOptimizationTest, TestLegacyTerms, TestQueryUtils, BadCopyFieldTest, TestLegacyFieldCache, TestConfigsApi, SimpleCollectionCreateDeleteTest, AnalyticsQueryTest, CdcrBootstrapTest, PKIAuthenticationIntegrationTest, TestPerFieldSimilarity, TestJsonRequest, TestEmbeddedSolrServerSchemaAPI, TestManagedStopFilterFactory, DistribCursorPagingTest, BasicDistributedZkTest, OverseerTest, ShardRoutingTest, BasicZkTest, LeaderElectionTest, TestRandomDVFaceting, ZkCLITest, ShardRoutingCustomTest, TermVectorComponentDistributedTest, ZkControllerTest, TestRealTimeGet, TestReload, TestCoreContainer, SpellCheckComponentTest, SolrCmdDistributorTest, QueryElevationComponentTest, TestSort, TestLazyCores, AnalysisAfterCoreReloadTest, SpellCheckCollatorTest, TestFoldingMultitermQuery, SuggesterTSTTest, SuggesterTest, SpatialFilterTest, TestPseudoReturnFields, TestAtomicUpdateErrorCases, StatelessScriptUpdateProcessorFactoryTest, XsltUpdateRequestHandlerTest, IndexBasedSpellCheckerTest, LukeRequestHandlerTest, PrimitiveFieldTypeTest, TestOmitPositions, XmlUpdateRequestHandlerTest, DocumentBuilderTest, TestValueSourceCache, PathHierarchyTokenizerFactoryTest, TermVectorComponentTest, MoreLikeThisHandlerTest, FieldAnalysisRequestHandlerTest, TestCollationField, QueryParsingTest, CSVRequestHandlerTest, SOLR749Test, TestSolrIndexConfig, ResponseLogComponentTest, BadComponentTest, TestMergePolicyConfig, MinimalSchemaTest, OutputWriterTest, TestFuzzyAnalyzedSuggestions, TestPostingsSolrHighlighter, SpellPossibilityIteratorTest, EchoParamsTest, ScriptEngineTest, DateMathParserTest, DateFieldTest, UUIDFieldTest, TestLuceneIndexBackCompat, ActionThrottleTest, ChaosMonkeyNothingIsSafeWithPullReplicasTest, MoveReplicaHDFSTest]
[junit4] Completed [574/783 (1!)] on J1 in 338.49s, 4 tests, 1 failure, 1 error, 1 skipped <<< FAILURES!
[...truncated 50144 lines...]
[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1497 - Failure
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1497/
1 tests failed.
FAILED: org.apache.lucene.index.TestIndexSorting.testRandom3
Error Message:
GC overhead limit exceeded
Stack Trace:
java.lang.OutOfMemoryError: GC overhead limit exceeded
at __randomizedtesting.SeedInfo.seed([BFE5392CDD7DC64C:1D3D77F6B98FEF4A]:0)
at org.apache.lucene.index.TestIndexSorting.testRandom3(TestIndexSorting.java:2275)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
Build Log:
[...truncated 1613 lines...]
[junit4] Suite: org.apache.lucene.index.TestIndexSorting
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestIndexSorting -Dtests.method=testRandom3 -Dtests.seed=BFE5392CDD7DC64C -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=fr-CA -Dtests.timezone=Europe/Kiev -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] ERROR 525s J1 | TestIndexSorting.testRandom3 <<<
[junit4] > Throwable #1: java.lang.OutOfMemoryError: GC overhead limit exceeded
[junit4] > at __randomizedtesting.SeedInfo.seed([BFE5392CDD7DC64C:1D3D77F6B98FEF4A]:0)
[junit4] > at org.apache.lucene.index.TestIndexSorting.testRandom3(TestIndexSorting.java:2275)
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J1/temp/lucene.index.TestIndexSorting_BFE5392CDD7DC64C-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {sparse_text=PostingsFormat(name=LuceneVarGapDocFreqInterval), docs=PostingsFormat(name=MockRandom), norms=PostingsFormat(name=MockRandom), positions=PostingsFormat(name=LuceneVarGapDocFreqInterval), id=PostingsFormat(name=MockRandom), term_vectors=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128)))}, docValues:{multi_valued_long=DocValuesFormat(name=Lucene70), foo=DocValuesFormat(name=Lucene70), numeric=DocValuesFormat(name=Memory), float=DocValuesFormat(name=Lucene70), multi_valued_numeric=DocValuesFormat(name=Direct), long=DocValuesFormat(name=Lucene70), points=DocValuesFormat(name=Direct), sparse_int=DocValuesFormat(name=Lucene70), bar=DocValuesFormat(name=Direct), multi_valued_double=DocValuesFormat(name=Direct), docs=DocValuesFormat(name=Direct), id=DocValuesFormat(name=Direct), multi_valued_int=DocValuesFormat(name=Memory), multi_valued_bytes=DocValuesFormat(name=Memory), sparse_text=DocValuesFormat(name=Lucene70), double=DocValuesFormat(name=Memory), sparse_binary=DocValuesFormat(name=Lucene70), positions=DocValuesFormat(name=Lucene70), int=DocValuesFormat(name=Direct), sorted=DocValuesFormat(name=Memory), sparse=DocValuesFormat(name=Lucene70), multi_valued_string=DocValuesFormat(name=Direct), norms=DocValuesFormat(name=Direct), bytes=DocValuesFormat(name=Direct), dense_int=DocValuesFormat(name=Direct), binary=DocValuesFormat(name=Memory), multi_valued_float=DocValuesFormat(name=Lucene70), term_vectors=DocValuesFormat(name=Memory)}, maxPointsInLeafNode=1596, maxMBSortInHeap=7.463460795761551, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@359d9135), locale=fr-CA, timezone=Europe/Kiev
[junit4] 2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=344442888,total=409468928
[junit4] 2> NOTE: All tests run in this JVM: [TestOmitPositions, TestNRTReaderCleanup, TestExternalCodecs, TestLongRangeFieldQueries, TestOrdinalMap, TestPhraseQuery, TestIndexSearcher, TestMatchAllDocsQuery, TestStringHelper, TestIsCurrent, TestBoostQuery, TestMultiDocValues, TestSubScorerFreqs, TestWeakIdentityMap, TestNGramPhraseQuery, TestBytesRefAttImpl, TestNoMergePolicy, TestMutablePointsReaderUtils, TestBooleanQueryVisitSubscorers, TestToken, TestParallelReaderEmptyIndex, TestSimpleAttributeImpl, TestTermVectors, TestConjunctions, TestSimilarity, TestSearch, TestStressDeletes, TestTerm, TestPointQueries, TestBufferedChecksum, TestTermVectorsReader, TestGraphTokenizers, TestBooleanQuery, TestAutomaton, TestCodecHoldsOpenFiles, TestSingleInstanceLockFactory, TestPerFieldDocValuesFormat, TestNorms, TestRegexpRandom, TestPrefixQuery, TestPrefixInBooleanQuery, TestLockFactory, TestIndexWriterExceptions2, TestIntArrayDocIdSet, TestField, TestIndexWriterDelete, TestTopFieldCollectorEarlyTermination, TestPolygon2D, TestSortRescorer, TestBinaryDocument, TestNewestSegment, TermInSetQueryTest, TestLSBRadixSorter, TestParallelLeafReader, TestCharArraySet, TestAllFilesHaveChecksumFooter, TestDuelingCodecs, TestFastDecompressionMode, TestDocumentsWriterStallControl, TestSloppyMath, TestReqOptSumScorer, TestStressNRT, TestFieldReuse, TestRAMDirectory, TestIndexWriterLockRelease, Test2BSortedDocValuesOrds, TestCrash, TestAxiomaticF2LOG, TestExceedMaxTermLength, TestMultiset, TestBytesRefArray, TestReaderWrapperDVTypeCheck, TestComplexExplanations, TestSearcherManager, TestBlockMaxConjunction, TestIndexManyDocuments, TestAtomicUpdate, TestCachingTokenFilter, TestTimSorterWorstCase, TestGeoEncodingUtils, TestIndexingSequenceNumbers, TestStressIndexing, TestPerFieldPostingsFormat, TestSort, TestFieldsReader, TestSimpleSearchEquivalence, TestTermsEnum2, Test2BDocs, TestIndexWriterOnDiskFull, TestDisjunctionMaxQuery, TestForceMergeForever, TestOmitTf, TestNearSpansOrdered, TestSameScoresWithThreads, TestMultiFields, TestTermRangeQuery, TestDocumentsWriterDeleteQueue, TestCrashCausesCorruptIndex, TestScorerPerf, TestFuzzyQuery, TestPayloads, TestDoc, TestWildcard, TestSimpleExplanationsOfNonMatches, TestRegexpQuery, TestPerSegmentDeletes, TestUniqueTermCount, TestUnicodeUtil, TestSortRandom, TestTopFieldCollector, TestSumDocFreq, TestSearchForDuplicates, TestRecyclingIntBlockAllocator, TestPositionIncrement, TestVirtualMethod, TestSimilarityProvider, TestPriorityQueue, TestDocCount, TestRecyclingByteBlockAllocator, TestRamUsageEstimator, TestNoMergeScheduler, TestByteBlockPool, TestBytesRef, TestDemo, TestCharFilter, TestRollback, TestNot, TestIndexWriterOnJRECrash, TestReaderClosed, MultiCollectorTest, TestScoreCachingWrappingScorer, TestIntsRef, Test2BPositions, TestIOUtils, Test2BTerms, TestByteArrayDataInput, Test2BBinaryDocValues, TestCharArrayMap, TestCharacterUtils, TestDelegatingAnalyzerWrapper, TestReusableStringReader, TestCodecUtil, TestBlockPostingsFormat2, TestBlockPostingsFormat3, TestForUtil, TestLucene50CompoundFormat, TestLucene50TermVectorsFormat, TestLucene60PointsFormat, TestIndexedDISI, TestLucene70NormsFormat, TestExitableDirectoryReader, TestFilterDirectoryReader, TestIndexOptions, TestIndexSorting]
[junit4] Completed [404/486 (1!)] on J1 in 557.02s, 50 tests, 1 error <<< FAILURES!
[...truncated 282 lines...]
[junit4] JVM J1: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp/junit4-J1-20180307_200513_2506295786228740793275.sysout
[junit4] >>> JVM J1 emitted unexpected output (verbatim) ----
[junit4] java.lang.OutOfMemoryError: GC overhead limit exceeded
[junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps/java_pid18775.hprof ...
[junit4] Heap dump file created [464769052 bytes in 1.937 secs]
[junit4] <<< JVM J1: EOF ----
[...truncated 22611 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:636: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:570: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid18775.hprof
Total time: 265 minutes 11 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1496 - Still
Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1496/
3 tests failed.
FAILED: org.apache.lucene.index.TestIndexSorting.testRandom3
Error Message:
Test abandoned because suite timeout was reached.
Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
at __randomizedtesting.SeedInfo.seed([2AB9BAC360DC26AE]:0)
FAILED: junit.framework.TestSuite.org.apache.lucene.index.TestIndexSorting
Error Message:
Suite timeout exceeded (>= 7200000 msec).
Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
at __randomizedtesting.SeedInfo.seed([2AB9BAC360DC26AE]:0)
FAILED: org.apache.solr.client.solrj.request.TestV1toV2ApiMapper.testCommands
Error Message:
null value is not valid for name
Stack Trace:
java.lang.RuntimeException: null value is not valid for name
at __randomizedtesting.SeedInfo.seed([669571DD12ACC28E:47A9B825789622EA]:0)
at org.apache.solr.client.solrj.request.V1toV2ApiMapper$ActionInfo.lambda$null$2(V1toV2ApiMapper.java:108)
at org.apache.solr.common.MapWriter.toMap(MapWriter.java:42)
at org.apache.solr.common.MapWriter$1.put(MapWriter.java:45)
at org.apache.solr.client.solrj.request.V1toV2ApiMapper$ActionInfo.lambda$serializeToV2Format$3(V1toV2ApiMapper.java:98)
at org.apache.solr.common.MapWriter.toMap(MapWriter.java:42)
at org.apache.solr.client.solrj.request.V1toV2ApiMapper$ActionInfo.convert(V1toV2ApiMapper.java:87)
at org.apache.solr.client.solrj.request.V1toV2ApiMapper.convert(V1toV2ApiMapper.java:137)
at org.apache.solr.client.solrj.request.TestV1toV2ApiMapper.testCommands(TestV1toV2ApiMapper.java:55)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 1985 lines...]
[junit4] Suite: org.apache.lucene.index.TestIndexSorting
[junit4] 2> mars 06, 2018 12:21:58 PM com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
[junit4] 2> WARNING: Suite execution timed out: org.apache.lucene.index.TestIndexSorting
[junit4] 2> 1) Thread[id=23660, name=Lucene Merge Thread #10293, state=RUNNABLE, group=TGRP-TestIndexSorting]
[junit4] 2> at org.apache.lucene.store.MockDirectoryWrapper.maybeThrowDeterministicException(MockDirectoryWrapper.java:1019)
[junit4] 2> at org.apache.lucene.store.MockIndexOutputWrapper.writeBytes(MockIndexOutputWrapper.java:144)
[junit4] 2> at org.apache.lucene.store.MockIndexOutputWrapper.writeByte(MockIndexOutputWrapper.java:126)
[junit4] 2> at org.apache.lucene.store.RateLimitedIndexOutput.writeByte(RateLimitedIndexOutput.java:66)
[junit4] 2> at org.apache.lucene.codecs.simpletext.SimpleTextUtil.write(SimpleTextUtil.java:48)
[junit4] 2> at org.apache.lucene.codecs.simpletext.SimpleTextUtil.write(SimpleTextUtil.java:39)
[junit4] 2> at org.apache.lucene.codecs.simpletext.SimpleTextDocValuesWriter.addNumericField(SimpleTextDocValuesWriter.java:140)
[junit4] 2> at org.apache.lucene.codecs.DocValuesConsumer.mergeNumericField(DocValuesConsumer.java:175)
[junit4] 2> at org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:135)
[junit4] 2> at org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:194)
[junit4] 2> at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:149)
[junit4] 2> at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4441)
[junit4] 2> at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4081)
[junit4] 2> at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:624)
[junit4] 2> at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:661)
[junit4] 2> 2) Thread[id=23643, name=Lucene Merge Thread #10276, state=RUNNABLE, group=TGRP-TestIndexSorting]
[junit4] 2> at org.apache.lucene.store.MockIndexInputWrapper.seek(MockIndexInputWrapper.java:128)
[junit4] 2> at org.apache.lucene.codecs.simpletext.SimpleTextDocValuesReader$3.advance(SimpleTextDocValuesReader.java:253)
[junit4] 2> at org.apache.lucene.codecs.simpletext.SimpleTextDocValuesReader$3.nextDoc(SimpleTextDocValuesReader.java:237)
[junit4] 2> at org.apache.lucene.codecs.simpletext.SimpleTextDocValuesReader$1.nextDoc(SimpleTextDocValuesReader.java:152)
[junit4] 2> at org.apache.lucene.codecs.DocValuesConsumer$NumericDocValuesSub.nextDoc(DocValuesConsumer.java:164)
[junit4] 2> at org.apache.lucene.index.DocIDMerger$SequentialDocIDMerger.next(DocIDMerger.java:99)
[junit4] 2> at org.apache.lucene.codecs.DocValuesConsumer$1$1.nextDoc(DocValuesConsumer.java:216)
[junit4] 2> at org.apache.lucene.codecs.simpletext.SimpleTextDocValuesWriter.addNumericField(SimpleTextDocValuesWriter.java:90)
[junit4] 2> at org.apache.lucene.codecs.DocValuesConsumer.mergeNumericField(DocValuesConsumer.java:175)
[junit4] 2> at org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:135)
[junit4] 2> at org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:194)
[junit4] 2> at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:149)
[junit4] 2> at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4441)
[junit4] 2> at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4081)
[junit4] 2> at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:624)
[junit4] 2> at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:661)
[junit4] 2> 3) Thread[id=23655, name=Lucene Merge Thread #10288, state=RUNNABLE, group=TGRP-TestIndexSorting]
[junit4] 2> at java.lang.Thread.yield(Native Method)
[junit4] 2> at org.apache.lucene.store.MockIndexOutputWrapper.writeBytes(MockIndexOutputWrapper.java:138)
[junit4] 2> at org.apache.lucene.store.MockIndexOutputWrapper.writeByte(MockIndexOutputWrapper.java:126)
[junit4] 2> at org.apache.lucene.store.RateLimitedIndexOutput.writeByte(RateLimitedIndexOutput.java:66)
[junit4] 2> at org.apache.lucene.codecs.simpletext.SimpleTextDocValuesWriter.doAddBinaryField(SimpleTextDocValuesWriter.java:208)
[junit4] 2> at org.apache.lucene.codecs.simpletext.SimpleTextDocValuesWriter.addSortedNumericField(SimpleTextDocValuesWriter.java:315)
[junit4] 2> at org.apache.lucene.codecs.DocValuesConsumer.mergeSortedNumericField(DocValuesConsumer.java:371)
[junit4] 2> at org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:143)
[junit4] 2> at org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:194)
[junit4] 2> at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:149)
[junit4] 2> at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4441)
[junit4] 2> at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4081)
[junit4] 2> at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:624)
[junit4] 2> at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:661)
[junit4] 2> 4) Thread[id=12487, name=TEST-TestIndexSorting.testRandom3-seed#[2AB9BAC360DC26AE], state=TIMED_WAITING, group=TGRP-TestIndexSorting]
[junit4] 2> at java.lang.Object.wait(Native Method)
[junit4] 2> at org.apache.lucene.index.ConcurrentMergeScheduler.doStall(ConcurrentMergeScheduler.java:616)
[junit4] 2> at org.apache.lucene.index.ConcurrentMergeScheduler.maybeStall(ConcurrentMergeScheduler.java:602)
[junit4] 2> at org.apache.lucene.index.ConcurrentMergeScheduler.merge(ConcurrentMergeScheduler.java:524)
[junit4] 2> at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2245)
[junit4] 2> at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5096)
[junit4] 2> at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1730)
[junit4] 2> at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1462)
[junit4] 2> at org.apache.lucene.index.TestIndexSorting.testRandom3(TestIndexSorting.java:2294)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2> at java.lang.reflect.Method.invoke(Method.java:498)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
[junit4] 2> at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 5) Thread[id=12486, name=SUITE-TestIndexSorting-seed#[2AB9BAC360DC26AE], state=RUNNABLE, group=TGRP-TestIndexSorting]
[junit4] 2> at java.lang.Thread.getStackTrace(Thread.java:1559)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:696)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:693)
[junit4] 2> at java.security.AccessController.doPrivileged(Native Method)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.getStackTrace(ThreadLeakControl.java:693)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.getThreadsWithTraces(ThreadLeakControl.java:709)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.formatThreadStacksFull(ThreadLeakControl.java:689)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.access$1000(ThreadLeakControl.java:65)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$2.evaluate(ThreadLeakControl.java:415)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:705)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.access$200(RandomizedRunner.java:139)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$2.run(RandomizedRunner.java:626)
[junit4] 2> 6) Thread[id=1, name=main, state=WAITING, group=main]
[junit4] 2> at java.lang.Object.wait(Native Method)
[junit4] 2> at java.lang.Thread.join(Thread.java:1252)
[junit4] 2> at java.lang.Thread.join(Thread.java:1326)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:636)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:493)
[junit4] 2> at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:251)
[junit4] 2> at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
[junit4] 2> at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
[junit4] 2> 7) Thread[id=9, name=JUnit4-serializer-daemon, state=TIMED_WAITING, group=main]
[junit4] 2> at java.lang.Thread.sleep(Native Method)
[junit4] 2> at com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:50)
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestIndexSorting -Dtests.method=testRandom3 -Dtests.seed=2AB9BAC360DC26AE -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=fr-LU -Dtests.timezone=America/Coral_Harbour -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
[junit4] ERROR 7181s J1 | TestIndexSorting.testRandom3 <<<
[junit4] > Throwable #1: java.lang.Exception: Test abandoned because suite timeout was reached.
[junit4] > at __randomizedtesting.SeedInfo.seed([2AB9BAC360DC26AE]:0)
[junit4] 2> mars 06, 2018 12:21:58 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 4 leaked thread(s).
[junit4] 2> mars 06, 2018 12:22:18 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> SEVERE: 4 threads leaked from SUITE scope at org.apache.lucene.index.TestIndexSorting:
[junit4] 2> 1) Thread[id=23675, name=Lucene Merge Thread #10308, state=RUNNABLE, group=TGRP-TestIndexSorting]
[junit4] 2> at org.apache.lucene.codecs.simpletext.SimpleTextUtil.readLine(SimpleTextUtil.java:70)
[junit4] 2> at org.apache.lucene.codecs.simpletext.SimpleTextDocValuesReader$6.advance(SimpleTextDocValuesReader.java:389)
[junit4] 2> at org.apache.lucene.codecs.simpletext.SimpleTextDocValuesReader$6.nextDoc(SimpleTextDocValuesReader.java:361)
[junit4] 2> at org.apache.lucene.codecs.simpletext.SimpleTextDocValuesReader$5.nextDoc(SimpleTextDocValuesReader.java:319)
[junit4] 2> at org.apache.lucene.codecs.simpletext.SimpleTextDocValuesReader$8.nextDoc(SimpleTextDocValuesReader.java:525)
[junit4] 2> at org.apache.lucene.codecs.DocValuesConsumer$SortedNumericDocValuesSub.nextDoc(DocValuesConsumer.java:359)
[junit4] 2> at org.apache.lucene.index.DocIDMerger$SequentialDocIDMerger.next(DocIDMerger.java:99)
[junit4] 2> at org.apache.lucene.codecs.DocValuesConsumer$3$1.nextDoc(DocValuesConsumer.java:414)
[junit4] 2> at org.apache.lucene.codecs.simpletext.SimpleTextDocValuesWriter$1$1.nextDoc(SimpleTextDocValuesWriter.java:323)
[junit4] 2> at org.apache.lucene.codecs.simpletext.SimpleTextDocValuesWriter.doAddBinaryField(SimpleTextDocValuesWriter.java:165)
[junit4] 2> at org.apache.lucene.codecs.simpletext.SimpleTextDocValuesWriter.addSortedNumericField(SimpleTextDocValuesWriter.java:315)
[junit4] 2> at org.apache.lucene.codecs.DocValuesConsumer.mergeSortedNumericField(DocValuesConsumer.java:371)
[junit4] 2> at org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:143)
[junit4] 2> at org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:194)
[junit4] 2> at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:149)
[junit4] 2> at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4441)
[junit4] 2> at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4081)
[junit4] 2> at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:624)
[junit4] 2> at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:661)
[junit4] 2> 2) Thread[id=23643, name=Lucene Merge Thread #10276, state=RUNNABLE, group=TGRP-TestIndexSorting]
[junit4] 2> at org.apache.lucene.store.MockIndexInputWrapper.seek(MockIndexInputWrapper.java:128)
[junit4] 2> at org.apache.lucene.codecs.simpletext.SimpleTextDocValuesReader$6.advance(SimpleTextDocValuesReader.java:377)
[junit4] 2> at org.apache.lucene.codecs.simpletext.SimpleTextDocValuesReader$6.nextDoc(SimpleTextDocValuesReader.java:361)
[junit4] 2> at org.apache.lucene.codecs.simpletext.SimpleTextDocValuesReader$5.nextDoc(SimpleTextDocValuesReader.java:319)
[junit4] 2> at org.apache.lucene.codecs.simpletext.SimpleTextDocValuesReader$8.nextDoc(SimpleTextDocValuesReader.java:525)
[junit4] 2> at org.apache.lucene.codecs.DocValuesConsumer$SortedNumericDocValuesSub.nextDoc(DocValuesConsumer.java:359)
[junit4] 2> at org.apache.lucene.index.DocIDMerger$SequentialDocIDMerger.next(DocIDMerger.java:99)
[junit4] 2> at org.apache.lucene.codecs.DocValuesConsumer$3$1.nextDoc(DocValuesConsumer.java:414)
[junit4] 2> at org.apache.lucene.codecs.simpletext.SimpleTextDocValuesWriter$1$1.nextDoc(SimpleTextDocValuesWriter.java:323)
[junit4] 2> at org.apache.lucene.codecs.simpletext.SimpleTextDocValuesWriter.doAddBinaryField(SimpleTextDocValuesWriter.java:165)
[junit4] 2> at org.apache.lucene.codecs.simpletext.SimpleTextDocValuesWriter.addSortedNumericField(SimpleTextDocValuesWriter.java:315)
[junit4] 2> at org.apache.lucene.codecs.DocValuesConsumer.mergeSortedNumericField(DocValuesConsumer.java:371)
[junit4] 2> at org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:143)
[junit4] 2> at org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:194)
[junit4] 2> at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:149)
[junit4] 2> at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4441)
[junit4] 2> at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4081)
[junit4] 2> at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:624)
[junit4] 2> at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:661)
[junit4] 2> 3) Thread[id=12487, name=TEST-TestIndexSorting.testRandom3-seed#[2AB9BAC360DC26AE], state=TIMED_WAITING, group=TGRP-TestIndexSorting]
[junit4] 2> at java.lang.Object.wait(Native Method)
[junit4] 2> at org.apache.lucene.index.ConcurrentMergeScheduler.doStall(ConcurrentMergeScheduler.java:616)
[junit4] 2> at org.apache.lucene.index.ConcurrentMergeScheduler.maybeStall(ConcurrentMergeScheduler.java:602)
[junit4] 2> at org.apache.lucene.index.ConcurrentMergeScheduler.merge(ConcurrentMergeScheduler.java:524)
[junit4] 2> at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2245)
[junit4] 2> at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5096)
[junit4] 2> at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1730)
[junit4] 2> at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1462)
[junit4] 2> at org.apache.lucene.index.TestIndexSorting.testRandom3(TestIndexSorting.java:2294)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2> at java.lang.reflect.Method.invoke(Method.java:498)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
[junit4] 2> at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 4) Thread[id=23679, name=Lucene Merge Thread #10312, state=RUNNABLE, group=TGRP-TestIndexSorting]
[junit4] 2> at org.apache.lucene.store.MockIndexInputWrapper.getFilePointer(MockIndexInputWrapper.java:122)
[junit4] 2> at org.apache.lucene.store.BufferedChecksumIndexInput.getFilePointer(BufferedChecksumIndexInput.java:65)
[junit4] 2> at org.apache.lucene.codecs.simpletext.SimpleTextDocValuesReader.checkIntegrity(SimpleTextDocValuesReader.java:729)
[junit4] 2> at org.apache.lucene.index.CodecReader.checkIntegrity(CodecReader.java:294)
[junit4] 2> at org.apache.lucene.index.MergeReaderWrapper.checkIntegrity(MergeReaderWrapper.java:179)
[junit4] 2> at org.apache.lucene.index.FilterLeafReader.checkIntegrity(FilterLeafReader.java:413)
[junit4] 2> at org.apache.lucene.index.SlowCodecReaderWrapper.wrap(SlowCodecReaderWrapper.java:54)
[junit4] 2> at org.apache.lucene.index.MockRandomMergePolicy$MockRandomOneMerge.wrapForMerge(MockRandomMergePolicy.java:159)
[junit4] 2> at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4426)
[junit4] 2> at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4081)
[junit4] 2> at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:624)
[junit4] 2> at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:661)
[junit4] 2> mars 06, 2018 12:22:18 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
[junit4] 2> INFO: Starting to interrupt leaked threads:
[junit4] 2> 1) Thread[id=23675, name=Lucene Merge Thread #10308, state=RUNNABLE, group=TGRP-TestIndexSorting]
[junit4] 2> 2) Thread[id=23643, name=Lucene Merge Thread #10276, state=RUNNABLE, group=TGRP-TestIndexSorting]
[junit4] 2> 3) Thread[id=12487, name=TEST-TestIndexSorting.testRandom3-seed#[2AB9BAC360DC26AE], state=TIMED_WAITING, group=TGRP-TestIndexSorting]
[junit4] 2> 4) Thread[id=23679, name=Lucene Merge Thread #10312, state=RUNNABLE, group=TGRP-TestIndexSorting]
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J1/temp/lucene.index.TestIndexSorting_2AB9BAC360DC26AE-001
[junit4] 2> Mar 06, 2018 5:22:19 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
[junit4] 2> INFO: All leaked threads terminated.
[junit4] 2> NOTE: test params are: codec=SimpleText, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@529c7d8c), locale=fr-LU, timezone=America/Coral_Harbour
[junit4] 2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=131949864,total=527433728
[junit4] 2> NOTE: All tests run in this JVM: [TestFilterWeight, Test2BTerms, TestFSTs, TestPrefixQuery, TestRecyclingByteBlockAllocator, TestDemoParallelLeafReader, TestIndexWriterUnicode, Test2BPostings, TestPerFieldDocValuesFormat, TestIndexWriterExceptions2, TestSearchForDuplicates, TestSortRandom, TestPrefixCodedTerms, TestDistributionSPL, TestWildcard, TestFixedBitDocIdSet, TestSimilarityProvider, TestTieredMergePolicy, TestSimilarity, TestBooleanQueryVisitSubscorers, TestIsCurrent, TestExternalCodecs, TestReusableStringReader, TestIndexWriterOnVMError, TestPolygon2D, TestPhraseQuery, TestNGramPhraseQuery, TestMinShouldMatch2, TestSubScorerFreqs, TestCachingTokenFilter, TestManyFields, TestSpanExplanations, TestIndexWriterDelete, TestIndexingSequenceNumbers, TestSearcherManager, TestDeterminism, TestRadixSelector, TestExceedMaxTermLength, TestStringHelper, TestByteSlices, TestLucene70SegmentInfoFormat, TestBoostQuery, TestBlockPostingsFormat2, TestDocValues, TestCodecHoldsOpenFiles, TestConcurrentMergeScheduler, TestSortedSetSortField, TestRegexpRandom2, TestTransactions, TestSameScoresWithThreads, TestGraphTokenizers, TestSimpleAttributeImpl, TestBytesRefAttImpl, TermInSetQueryTest, TestLockFactory, TestDoubleValuesSource, TestDocValuesQueries, TestAutomaton, TestNorms, TestStressNRT, TestTimeLimitingCollector, TestTimSorterWorstCase, TestLucene50TermVectorsFormat, TestRAMDirectory, TestStressDeletes, TestFieldReuse, TestTermVectors, TestLongRangeFieldQueries, TestBooleanQuery, TestMutablePointsReaderUtils, TestMatchAllDocsQuery, TestMultiset, TestIndexSorting]
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestIndexSorting -Dtests.seed=2AB9BAC360DC26AE -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=fr-LU -Dtests.timezone=America/Coral_Harbour -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
[junit4] ERROR 0.00s J1 | TestIndexSorting (suite) <<<
[junit4] > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
[junit4] > at __randomizedtesting.SeedInfo.seed([2AB9BAC360DC26AE]:0)
[junit4] Completed [478/486 (1!)] on J1 in 7220.84s, 36 tests, 2 errors <<< FAILURES!
[...truncated 14268 lines...]
[junit4] Suite: org.apache.solr.client.solrj.request.TestV1toV2ApiMapper
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestV1toV2ApiMapper -Dtests.method=testCommands -Dtests.seed=669571DD12ACC28E -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=en-AU -Dtests.timezone=America/Barbados -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
[junit4] ERROR 0.02s J1 | TestV1toV2ApiMapper.testCommands <<<
[junit4] > Throwable #1: java.lang.RuntimeException: null value is not valid for name
[junit4] > at __randomizedtesting.SeedInfo.seed([669571DD12ACC28E:47A9B825789622EA]:0)
[junit4] > at org.apache.solr.client.solrj.request.V1toV2ApiMapper$ActionInfo.lambda$null$2(V1toV2ApiMapper.java:108)
[junit4] > at org.apache.solr.common.MapWriter.toMap(MapWriter.java:42)
[junit4] > at org.apache.solr.common.MapWriter$1.put(MapWriter.java:45)
[junit4] > at org.apache.solr.client.solrj.request.V1toV2ApiMapper$ActionInfo.lambda$serializeToV2Format$3(V1toV2ApiMapper.java:98)
[junit4] > at org.apache.solr.common.MapWriter.toMap(MapWriter.java:42)
[junit4] > at org.apache.solr.client.solrj.request.V1toV2ApiMapper$ActionInfo.convert(V1toV2ApiMapper.java:87)
[junit4] > at org.apache.solr.client.solrj.request.V1toV2ApiMapper.convert(V1toV2ApiMapper.java:137)
[junit4] > at org.apache.solr.client.solrj.request.TestV1toV2ApiMapper.testCommands(TestV1toV2ApiMapper.java:55)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] 2> NOTE: test params are: codec=Lucene70, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@7cc930ca), locale=en-AU, timezone=America/Barbados
[junit4] 2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=204165184,total=363855872
[junit4] 2> NOTE: All tests run in this JVM: [ConcatOperationTest, FloorEvaluatorTest, BasicHttpSolrClientTest, CumulativeProbabilityEvaluatorTest, RoundEvaluatorTest, TestDocumentObjectBinder, ModuloEvaluatorTest, JDBCStreamTest, UniformDistributionEvaluatorTest, SolrQueryTest, LengthEvaluatorTest, NormalDistributionEvaluatorTest, OrEvaluatorTest, SolrDocumentTest, HyperbolicCosineEvaluatorTest, SolrExampleEmbeddedTest, ConcurrentUpdateSolrClientTest, AbsoluteValueEvaluatorTest, CloudSolrClientMultiConstructorTest, CollectionAdminRequestRequiredParamsTest, EqualToEvaluatorTest, ArcTangentEvaluatorTest, CeilingEvaluatorTest, AppendEvaluatorTest, SolrExampleStreamingBinaryTest, SolrExampleBinaryTest, SolrExampleJettyTest, LargeVolumeEmbeddedTest, LargeVolumeJettyTest, SolrParamTest, TermsResponseTest, TestEmbeddedSolrServer, TestCoreAdmin, HttpSolrClientSSLAuthConPoolTest, GraphExpressionTest, JdbcTest, StreamExpressionToExpessionTest, StreamExpressionToExplanationTest, AddEvaluatorTest, AndEvaluatorTest, ArrayEvaluatorTest, CorrelationEvaluatorTest, EmpiricalDistributionEvaluatorTest, GreaterThanEqualToEvaluatorTest, LessThanEqualToEvaluatorTest, MultiplyEvaluatorTest, RecursiveEvaluatorTest, SquareRootEvaluatorTest, SubtractEvaluatorTest, TangentEvaluatorTest, UuidEvaluatorTest, StreamExpressionParserTest, OperationsTest, SolrPingTest, TestCollectionAdminRequest, TestDelegationTokenRequest, TestV1toV2ApiMapper]
[junit4] Completed [131/146 (1!)] on J1 in 0.04s, 1 test, 1 error <<< FAILURES!
[...truncated 48574 lines...]
[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1495 - Still
Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1495/
3 tests failed.
FAILED: org.apache.solr.cloud.TestStressLiveNodes.testStress
Error Message:
iter2268: [127.0.0.1:42398_solr, thrasher-T2267_0-0, thrasher-T2267_0-1, thrasher-T2267_0-2, thrasher-T2267_0-3, thrasher-T2267_0-4, thrasher-T2267_1-0, thrasher-T2267_1-1, thrasher-T2267_1-2, thrasher-T2267_1-3, thrasher-T2267_1-4] expected:<1> but was:<11>
Stack Trace:
java.lang.AssertionError: iter2268: [127.0.0.1:42398_solr, thrasher-T2267_0-0, thrasher-T2267_0-1, thrasher-T2267_0-2, thrasher-T2267_0-3, thrasher-T2267_0-4, thrasher-T2267_1-0, thrasher-T2267_1-1, thrasher-T2267_1-2, thrasher-T2267_1-3, thrasher-T2267_1-4] expected:<1> but was:<11>
at __randomizedtesting.SeedInfo.seed([D11AA6B8CC8BF8CD:C401DB794F7C00B6]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:472)
at org.apache.solr.cloud.TestStressLiveNodes.testStress(TestStressLiveNodes.java:140)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.hdfs.HdfsUnloadDistributedZkTest.test
Error Message:
Error from server at http://127.0.0.1:37819/_pb/mn: ADDREPLICA failed to create replica
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:37819/_pb/mn: ADDREPLICA failed to create replica
at __randomizedtesting.SeedInfo.seed([D11AA6B8CC8BF8CD:594E996262779535]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1104)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:884)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:817)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
at org.apache.solr.cloud.UnloadDistributedZkTest.testUnloadShardAndCollection(UnloadDistributedZkTest.java:125)
at org.apache.solr.cloud.UnloadDistributedZkTest.test(UnloadDistributedZkTest.java:72)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.MoveReplicaHDFSTest.testFailedMove
Error Message:
No live SolrServers available to handle this request:[https://127.0.0.1:34817/solr/MoveReplicaHDFSTest_failed_coll_true, https://127.0.0.1:49102/solr/MoveReplicaHDFSTest_failed_coll_true]
Stack Trace:
org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[https://127.0.0.1:34817/solr/MoveReplicaHDFSTest_failed_coll_true, https://127.0.0.1:49102/solr/MoveReplicaHDFSTest_failed_coll_true]
at __randomizedtesting.SeedInfo.seed([D11AA6B8CC8BF8CD:7BD7754A7B582D1D]:0)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:462)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1104)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:884)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:991)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:991)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:817)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942)
at org.apache.solr.cloud.MoveReplicaTest.testFailedMove(MoveReplicaTest.java:309)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:34817/solr/MoveReplicaHDFSTest_failed_coll_true: no servers hosting shard: shard2
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:436)
... 47 more
Build Log:
[...truncated 13647 lines...]
[junit4] Suite: org.apache.solr.cloud.TestStressLiveNodes
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestStressLiveNodes_D11AA6B8CC8BF8CD-001/init-core-data-001
[junit4] 2> 1599902 WARN (SUITE-TestStressLiveNodes-seed#[D11AA6B8CC8BF8CD]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
[junit4] 2> 1599903 INFO (SUITE-TestStressLiveNodes-seed#[D11AA6B8CC8BF8CD]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 1599905 INFO (SUITE-TestStressLiveNodes-seed#[D11AA6B8CC8BF8CD]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 1599905 INFO (SUITE-TestStressLiveNodes-seed#[D11AA6B8CC8BF8CD]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 1599905 INFO (SUITE-TestStressLiveNodes-seed#[D11AA6B8CC8BF8CD]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 1 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestStressLiveNodes_D11AA6B8CC8BF8CD-001/tempDir-001
[junit4] 2> 1599906 INFO (SUITE-TestStressLiveNodes-seed#[D11AA6B8CC8BF8CD]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 1599906 INFO (Thread-10038) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1599906 INFO (Thread-10038) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 1599909 ERROR (Thread-10038) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 1600006 INFO (SUITE-TestStressLiveNodes-seed#[D11AA6B8CC8BF8CD]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:48205
[junit4] 2> 1600009 INFO (zkConnectionManagerCallback-2521-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600015 INFO (jetty-launcher-2518-thread-1) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T06:27:37+09:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 1600016 INFO (jetty-launcher-2518-thread-1) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 1600017 INFO (jetty-launcher-2518-thread-1) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 1600017 INFO (jetty-launcher-2518-thread-1) [ ] o.e.j.s.session Scavenging every 600000ms
[junit4] 2> 1600017 INFO (jetty-launcher-2518-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3b0cd341{/solr,null,AVAILABLE}
[junit4] 2> 1600018 INFO (jetty-launcher-2518-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5182bb91{HTTP/1.1,[http/1.1]}{127.0.0.1:42398}
[junit4] 2> 1600018 INFO (jetty-launcher-2518-thread-1) [ ] o.e.j.s.Server Started @1603054ms
[junit4] 2> 1600018 INFO (jetty-launcher-2518-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=42398}
[junit4] 2> 1600018 ERROR (jetty-launcher-2518-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1600018 INFO (jetty-launcher-2518-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.0.0
[junit4] 2> 1600018 INFO (jetty-launcher-2518-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1600019 INFO (jetty-launcher-2518-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1600019 INFO (jetty-launcher-2518-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-03-05T18:06:00.221Z
[junit4] 2> 1600021 INFO (zkConnectionManagerCallback-2523-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600022 INFO (jetty-launcher-2518-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 1600034 INFO (jetty-launcher-2518-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:48205/solr
[junit4] 2> 1600035 INFO (zkConnectionManagerCallback-2527-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600039 INFO (zkConnectionManagerCallback-2529-thread-1-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600151 INFO (jetty-launcher-2518-thread-1) [n:127.0.0.1:42398_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1600152 INFO (jetty-launcher-2518-thread-1) [n:127.0.0.1:42398_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:42398_solr
[junit4] 2> 1600153 INFO (jetty-launcher-2518-thread-1) [n:127.0.0.1:42398_solr ] o.a.s.c.Overseer Overseer (id=73402724547493891-127.0.0.1:42398_solr-n_0000000000) starting
[junit4] 2> 1600162 INFO (jetty-launcher-2518-thread-1) [n:127.0.0.1:42398_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:42398_solr
[junit4] 2> 1600169 INFO (OverseerStateUpdate-73402724547493891-127.0.0.1:42398_solr-n_0000000000) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1600262 INFO (jetty-launcher-2518-thread-1) [n:127.0.0.1:42398_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_42398.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3d0f64b3
[junit4] 2> 1600276 INFO (jetty-launcher-2518-thread-1) [n:127.0.0.1:42398_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_42398.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3d0f64b3
[junit4] 2> 1600276 INFO (jetty-launcher-2518-thread-1) [n:127.0.0.1:42398_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_42398.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3d0f64b3
[junit4] 2> 1600278 INFO (jetty-launcher-2518-thread-1) [n:127.0.0.1:42398_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestStressLiveNodes_D11AA6B8CC8BF8CD-001/tempDir-001/node1/.
[junit4] 2> 1600317 INFO (zkConnectionManagerCallback-2533-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600328 INFO (zkConnectionManagerCallback-2537-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600330 INFO (SUITE-TestStressLiveNodes-seed#[D11AA6B8CC8BF8CD]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1600331 INFO (SUITE-TestStressLiveNodes-seed#[D11AA6B8CC8BF8CD]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:48205/solr ready
[junit4] 2> 1600343 INFO (TEST-TestStressLiveNodes.testStress-seed#[D11AA6B8CC8BF8CD]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testStress
[junit4] 2> 1600345 INFO (zkConnectionManagerCallback-2539-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600347 INFO (TEST-TestStressLiveNodes.testStress-seed#[D11AA6B8CC8BF8CD]) [ ] o.a.s.c.TestStressLiveNodes preparing parallel adds to live nodes: iter=0, numThreads=2 numNodesPerThread=5
[junit4] 2> 1600349 INFO (zkConnectionManagerCallback-2541-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600351 INFO (zkConnectionManagerCallback-2543-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600361 INFO (zkCallback-2528-thread-1-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (6)
[junit4] 2> 1600365 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (6)
[junit4] 2> 1600367 INFO (zkCallback-2528-thread-1-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (6) -> (7)
[junit4] 2> 1600367 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (6) -> (7)
[junit4] 2> 1600368 INFO (zkCallback-2528-thread-1-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (7) -> (8)
[junit4] 2> 1600369 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (7) -> (8)
[junit4] 2> 1600371 INFO (zkCallback-2528-thread-1-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (8) -> (9)
[junit4] 2> 1600372 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (8) -> (9)
[junit4] 2> 1600373 INFO (zkCallback-2528-thread-1-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (9) -> (10)
[junit4] 2> 1600374 INFO (zkCallback-2528-thread-1-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (10) -> (11)
[junit4] 2> 1600374 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (9) -> (11)
[junit4] 2> 1600377 INFO (zkConnectionManagerCallback-2545-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600379 INFO (zkCallback-2528-thread-1-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (11) -> (6)
[junit4] 2> 1600379 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (11) -> (1)
[junit4] 2> 1600381 INFO (zkCallback-2528-thread-2-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (6) -> (1)
[junit4] 2> 1600386 INFO (zkConnectionManagerCallback-2547-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600390 INFO (TEST-TestStressLiveNodes.testStress-seed#[D11AA6B8CC8BF8CD]) [ ] o.a.s.c.TestStressLiveNodes preparing parallel adds to live nodes: iter=1, numThreads=3 numNodesPerThread=2
[junit4] 2> 1600391 INFO (zkConnectionManagerCallback-2549-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600393 INFO (zkConnectionManagerCallback-2551-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600395 INFO (zkConnectionManagerCallback-2553-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600397 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1600397 INFO (zkCallback-2528-thread-2-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1600399 INFO (zkCallback-2528-thread-2-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1600399 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1600400 INFO (zkCallback-2528-thread-2-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1600401 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1600402 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (6)
[junit4] 2> 1600402 INFO (zkCallback-2528-thread-2-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (6)
[junit4] 2> 1600404 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (6) -> (7)
[junit4] 2> 1600404 INFO (zkCallback-2528-thread-2-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (6) -> (7)
[junit4] 2> 1600409 INFO (zkConnectionManagerCallback-2555-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600411 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (7) -> (5)
[junit4] 2> 1600411 INFO (zkCallback-2528-thread-2-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (7) -> (5)
[junit4] 2> 1600412 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x104c7639a33000c, likely client has closed socket
[junit4] 2> 1600412 INFO (zkCallback-2528-thread-1-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (1)
[junit4] 2> 1600413 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (1)
[junit4] 2> 1600417 INFO (zkConnectionManagerCallback-2557-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600418 INFO (TEST-TestStressLiveNodes.testStress-seed#[D11AA6B8CC8BF8CD]) [ ] o.a.s.c.TestStressLiveNodes preparing parallel adds to live nodes: iter=2, numThreads=2 numNodesPerThread=5
[junit4] 2> 1600421 INFO (zkConnectionManagerCallback-2559-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600422 INFO (zkConnectionManagerCallback-2561-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600424 INFO (zkCallback-2528-thread-1-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1600425 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1600425 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1600425 INFO (zkCallback-2528-thread-1-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1600427 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1600427 INFO (zkCallback-2528-thread-1-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1600427 INFO (zkCallback-2528-thread-1-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 1600428 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 1600429 INFO (zkCallback-2528-thread-1-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (7)
[junit4] 2> 1600429 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (7)
[junit4] 2> 1600431 INFO (zkCallback-2528-thread-1-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (7) -> (9)
[junit4] 2> 1600431 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (7) -> (9)
[junit4] 2> 1600433 INFO (zkCallback-2528-thread-1-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (9) -> (11)
[junit4] 2> 1600433 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (9) -> (11)
[junit4] 2> 1600435 INFO (zkConnectionManagerCallback-2563-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600437 INFO (zkCallback-2528-thread-1-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (11) -> (6)
[junit4] 2> 1600437 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (11) -> (6)
[junit4] 2> 1600438 INFO (zkCallback-2528-thread-2-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (6) -> (1)
[junit4] 2> 1600438 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (6) -> (1)
[junit4] 2> 1600441 INFO (zkConnectionManagerCallback-2565-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600442 INFO (TEST-TestStressLiveNodes.testStress-seed#[D11AA6B8CC8BF8CD]) [ ] o.a.s.c.TestStressLiveNodes preparing parallel adds to live nodes: iter=3, numThreads=3 numNodesPerThread=5
[junit4] 2> 1600443 INFO (zkConnectionManagerCallback-2567-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600445 INFO (zkConnectionManagerCallback-2569-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600446 INFO (zkConnectionManagerCallback-2571-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600449 INFO (zkCallback-2528-thread-2-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (4)
[junit4] 2> 1600449 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (4)
[junit4] 2> 1600452 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 1600452 INFO (zkCallback-2528-thread-2-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (7)
[junit4] 2> 1600454 INFO (zkCallback-2528-thread-2-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (7) -> (9)
[junit4] 2> 1600455 INFO (zkCallback-2528-thread-2-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (9) -> (10)
[junit4] 2> 1600456 INFO (zkCallback-2528-thread-2-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (10) -> (11)
[junit4] 2> 1600456 INFO (zkCallback-2528-thread-2-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (11) -> (12)
[junit4] 2> 1600457 INFO (zkCallback-2528-thread-2-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (12) -> (13)
[junit4] 2> 1600457 INFO (zkCallback-2536-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (13)
[junit4] 2> 1600458 INFO (zkCallback-2528-thread-2-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (13) -> (16)
[junit4] 2> 1600460 INFO (zkConnectionManagerCallback-2573-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600461 INFO (zkCallback-2536-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (13) -> (16)
[junit4] 2> 1600462 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x104c7639a330014, likely client has closed socket
[junit4] 2> 1600462 INFO (zkCallback-2528-thread-2-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (16) -> (11)
[junit4] 2> 1600462 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x104c7639a330015, likely client has closed socket
[junit4] 2> 1600463 INFO (zkCallback-2528-thread-1-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (11) -> (1)
[junit4] 2> 1600464 INFO (zkCallback-2536-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (16) -> (1)
[junit4] 2> 1600465 INFO (zkConnectionManagerCallback-2575-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600466 INFO (TEST-TestStressLiveNodes.testStress-seed#[D11AA6B8CC8BF8CD]) [ ] o.a.s.c.TestStressLiveNodes preparing parallel adds to live nodes: iter=4, numThreads=4 numNodesPerThread=4
[junit4] 2> 1600468 INFO (zkConnectionManagerCallback-2577-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600473 INFO (zkConnectionManagerCallback-2579-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600474 INFO (zkConnectionManagerCallback-2581-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600476 INFO (zkConnectionManagerCallback-2583-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600479 INFO (zkCallback-2528-thread-1-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1600479 INFO (zkCallback-2536-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1600481 INFO (zkCallback-2528-thread-1-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1600481 INFO (zkCallback-2536-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1600482 INFO (zkCallback-2536-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1600483 INFO (zkCallback-2536-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (6)
[junit4] 2> 1600486 INFO (zkCallback-2536-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (6) -> (8)
[junit4] 2> 1600487 INFO (zkCallback-2536-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (8) -> (9)
[junit4] 2> 1600488 INFO (zkCallback-2528-thread-1-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1600489 INFO (zkCallback-2528-thread-2-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (9)
[junit4] 2> 1600490 INFO (zkCallback-2536-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (9) -> (11)
[junit4] 2> 1600490 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (11) -> (13)
[junit4] 2> 1600490 INFO (zkCallback-2528-thread-2-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (9) -> (13)
[junit4] 2> 1600495 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (13) -> (15)
[junit4] 2> 1600495 INFO (zkCallback-2528-thread-2-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (13) -> (15)
[junit4] 2> 1600496 INFO (zkCallback-2528-thread-2-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (15) -> (16)
[junit4] 2> 1600496 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (15) -> (16)
[junit4] 2> 1600498 INFO (zkCallback-2528-thread-2-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (16) -> (17)
[junit4] 2> 1600498 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (16) -> (17)
[junit4] 2> 1600501 INFO (zkConnectionManagerCallback-2585-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600503 INFO (zkCallback-2528-thread-2-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (17) -> (13)
[junit4] 2> 1600504 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (17) -> (13)
[junit4] 2> 1600504 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x104c7639a33001a, likely client has closed socket
[junit4] 2> 1600505 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (13) -> (9)
[junit4] 2> 1600505 INFO (zkCallback-2528-thread-1-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (13) -> (9)
[junit4] 2> 1600506 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (9) -> (5)
[junit4] 2> 1600510 INFO (zkCallback-2528-thread-3-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (9) -> (1)
[junit4] 2> 1600510 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (1)
[junit4] 2> 1600514 INFO (zkConnectionManagerCallback-2587-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600516 INFO (TEST-TestStressLiveNodes.testStress-seed#[D11AA6B8CC8BF8CD]) [ ] o.a.s.c.TestStressLiveNodes preparing parallel adds to live nodes: iter=5, numThreads=3 numNodesPerThread=1
[junit4] 2> 1600517 INFO (zkConnectionManagerCallback-2589-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600519 INFO (zkConnectionManagerCallback-2591-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600520 INFO (zkConnectionManagerCallback-2593-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600523 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1600525 INFO (zkCallback-2528-thread-3-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1600527 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1600527 INFO (zkCallback-2528-thread-3-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1600529 INFO (zkCallback-2528-thread-3-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1600529 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1600531 INFO (zkConnectionManagerCallback-2595-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600533 INFO (zkCallback-2528-thread-3-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
[junit4] 2> 1600534 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
[junit4] 2> 1600535 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (2)
[junit4] 2> 1600535 INFO (zkCallback-2528-thread-1-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (2)
[junit4] 2> 1600537 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
[junit4] 2> 1600537 INFO (zkCallback-2528-thread-3-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
[junit4] 2> 1600539 INFO (zkConnectionManagerCallback-2597-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600541 INFO (TEST-TestStressLiveNodes.testStress-seed#[D11AA6B8CC8BF8CD]) [ ] o.a.s.c.TestStressLiveNodes preparing parallel adds to live nodes: iter=6, numThreads=5 numNodesPerThread=5
[junit4] 2> 1600542 INFO (zkConnectionManagerCallback-2599-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600544 INFO (zkConnectionManagerCallback-2601-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600546 INFO (zkConnectionManagerCallback-2603-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600547 INFO (zkConnectionManagerCallback-2605-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600549 INFO (zkConnectionManagerCallback-2607-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600552 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (3)
[junit4] 2> 1600552 INFO (zkCallback-2528-thread-3-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (4)
[junit4] 2> 1600553 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1600554 INFO (zkCallback-2528-thread-3-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (6)
[junit4] 2> 1600554 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (6)
[junit4] 2> 1600556 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (6) -> (7)
[junit4] 2> 1600556 INFO (zkCallback-2528-thread-3-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (6) -> (7)
[junit4] 2> 1600556 INFO (zkCallback-2528-thread-3-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (7) -> (8)
[junit4] 2> 1600557 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (7) -> (8)
[junit4] 2> 1600558 INFO (zkCallback-2528-thread-3-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (8) -> (11)
[junit4] 2> 1600558 INFO (zkCallback-2536-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (8) -> (11)
[junit4] 2> 1600559 INFO (zkCallback-2528-thread-3-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (11) -> (12)
[junit4] 2> 1600560 INFO (zkCallback-2536-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (11) -> (12)
[junit4] 2> 1600561 INFO (zkCallback-2528-thread-3-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (12) -> (16)
[junit4] 2> 1600561 INFO (zkCallback-2536-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (12) -> (16)
[junit4] 2> 1600565 INFO (zkCallback-2528-thread-3-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (16) -> (18)
[junit4] 2> 1600566 INFO (zkCallback-2536-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (16) -> (18)
[junit4] 2> 1600566 INFO (zkCallback-2528-thread-3-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (18) -> (19)
[junit4] 2> 1600566 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (18) -> (19)
[junit4] 2> 1600568 INFO (zkCallback-2528-thread-3-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (19) -> (21)
[junit4] 2> 1600568 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (19) -> (21)
[junit4] 2> 1600569 INFO (zkCallback-2528-thread-3-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (21) -> (22)
[junit4] 2> 1600570 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (21) -> (22)
[junit4] 2> 1600571 INFO (zkCallback-2528-thread-3-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (22) -> (23)
[junit4] 2> 1600571 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (22) -> (23)
[junit4] 2> 1600573 INFO (zkCallback-2528-thread-3-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (23) -> (24)
[junit4] 2> 1600573 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (23) -> (24)
[junit4] 2> 1600574 INFO (zkCallback-2528-thread-3-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (24) -> (25)
[junit4] 2> 1600575 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (24) -> (25)
[junit4] 2> 1600577 INFO (zkCallback-2528-thread-3-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (25) -> (26)
[junit4] 2> 1600577 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (25) -> (26)
[junit4] 2> 1600579 INFO (zkConnectionManagerCallback-2609-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600581 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x104c7639a330024, likely client has closed socket
[junit4] 2> 1600581 INFO (zkCallback-2528-thread-3-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (26) -> (21)
[junit4] 2> 1600582 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (26) -> (21)
[junit4] 2> 1600583 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x104c7639a330025, likely client has closed socket
[junit4] 2> 1600583 INFO (zkCallback-2528-thread-1-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (21) -> (16)
[junit4] 2> 1600583 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (21) -> (16)
[junit4] 2> 1600584 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (16) -> (11)
[junit4] 2> 1600585 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x104c7639a330027, likely client has closed socket
[junit4] 2> 1600586 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (11) -> (6)
[junit4] 2> 1600589 INFO (zkCallback-2528-thread-2-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (16) -> (6)
[junit4] 2> 1600589 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x104c7639a330028, likely client has closed socket
[junit4] 2> 1600590 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (6) -> (1)
[junit4] 2> 1600597 INFO (zkCallback-2528-thread-4-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (6) -> (1)
[junit4] 2> 1600597 INFO (zkConnectionManagerCallback-2611-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600599 INFO (TEST-TestStressLiveNodes.testStress-seed#[D11AA6B8CC8BF8CD]) [ ] o.a.s.c.TestStressLiveNodes preparing parallel adds to live nodes: iter=7, numThreads=2 numNodesPerThread=2
[junit4] 2> 1600601 INFO (zkConnectionManagerCallback-2613-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600604 INFO (zkConnectionManagerCallback-2615-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600606 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (3)
[junit4] 2> 1600606 INFO (zkCallback-2528-thread-4-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (3)
[junit4] 2> 1600609 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (5)
[junit4] 2> 1600609 INFO (zkCallback-2528-thread-4-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (5)
[junit4] 2> 1600610 INFO (zkConnectionManagerCallback-2617-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600612 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (3)
[junit4] 2> 1600613 INFO (zkCallback-2528-thread-4-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (3)
[junit4] 2> 1600613 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (1)
[junit4] 2> 1600614 INFO (zkCallback-2528-thread-2-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (1)
[junit4] 2> 1600614 INFO (zkConnectionManagerCallback-2619-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600616 INFO (TEST-TestStressLiveNodes.testStress-seed#[D11AA6B8CC8BF8CD]) [ ] o.a.s.c.TestStressLiveNodes preparing parallel adds to live nodes: iter=8, numThreads=3 numNodesPerThread=3
[junit4] 2> 1600617 INFO (zkConnectionManagerCallback-2621-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600619 INFO (zkConnectionManagerCallback-2623-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600621 INFO (zkConnectionManagerCallback-2625-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600623 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (3)
[junit4] 2> 1600623 INFO (zkCallback-2528-thread-2-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (3)
[junit4] 2> 1600624 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (5)
[junit4] 2> 1600625 INFO (zkCallback-2528-thread-2-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (5)
[junit4] 2> 1600626 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (8)
[junit4] 2> 1600627 INFO (zkCallback-2528-thread-2-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (8)
[junit4] 2> 1600628 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (8) -> (10)
[junit4] 2> 1600629 INFO (zkCallback-2528-thread-2-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (8) -> (10)
[junit4] 2> 1600630 INFO (zkConnectionManagerCallback-2627-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600633 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x104c7639a33002f, likely client has closed socket
[junit4] 2> 1600634 INFO (zkCallback-2528-thread-2-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (10) -> (7)
[junit4] 2> 1600634 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (10) -> (4)
[junit4] 2> 1600635 INFO (zkCallback-2528-thread-4-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (7) -> (1)
[junit4] 2> 1600635 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (1)
[junit4] 2> 1600636 INFO (zkConnectionManagerCallback-2629-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600638 INFO (TEST-TestStressLiveNodes.testStress-seed#[D11AA6B8CC8BF8CD]) [ ] o.a.s.c.TestStressLiveNodes preparing parallel adds to live nodes: iter=9, numThreads=2 numNodesPerThread=5
[junit4] 2> 1600639 INFO (zkConnectionManagerCallback-2631-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600640 INFO (zkConnectionManagerCallback-2633-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600642 INFO (zkCallback-2528-thread-4-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1600642 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1600643 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1600643 INFO (zkCallback-2528-thread-4-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1600644 INFO (zkCallback-2528-thread-4-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1600644 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1600645 INFO (zkCallback-2528-thread-4-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 1600646 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 1600646 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (6)
[junit4] 2> 1600647 INFO (zkCallback-2528-thread-4-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (6)
[junit4] 2> 1600647 INFO (zkCallback-2528-thread-4-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (6) -> (7)
[junit4] 2> 1600647 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (6) -> (7)
[junit4] 2> 1600649 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (7) -> (9)
[junit4] 2> 1600649 INFO (zkCallback-2528-thread-4-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (7) -> (9)
[junit4] 2> 1600650 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (9) -> (11)
[junit4] 2> 1600650 INFO (zkCallback-2528-thread-4-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (9) -> (11)
[junit4] 2> 1600651 INFO (zkConnectionManagerCallback-2635-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600653 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (11) -> (6)
[junit4] 2> 1600653 INFO (zkCallback-2528-thread-4-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (11) -> (6)
[junit4] 2> 1600654 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (6) -> (1)
[junit4] 2> 1600654 INFO (zkCallback-2528-thread-2-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (6) -> (1)
[junit4] 2> 1600655 INFO (zkConnectionManagerCallback-2637-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600657 INFO (TEST-TestStressLiveNodes.testStress-seed#[D11AA6B8CC8BF8CD]) [ ] o.a.s.c.TestStressLiveNodes preparing parallel adds to live nodes: iter=10, numThreads=5 numNodesPerThread=4
[junit4] 2> 1600658 INFO (zkConnectionManagerCallback-2639-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600659 INFO (zkConnectionManagerCallback-2641-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600664 INFO (zkConnectionManagerCallback-2643-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600666 INFO (zkConnectionManagerCallback-2645-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600668 INFO (zkConnectionManagerCallback-2647-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600670 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1600670 INFO (zkCallback-2528-thread-4-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1600671 INFO (zkCallback-2528-thread-4-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (4)
[junit4] 2> 1600672 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (4)
[junit4] 2> 1600673 INFO (zkCallback-2528-thread-4-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 1600673 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 1600674 INFO (zkCallback-2528-thread-4-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (7)
[junit4] 2> 1600674 INFO (zkCallback-2536-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (7)
[junit4] 2> 1600675 INFO (zkCallback-2536-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (7) -> (10)
[junit4] 2> 1600676 INFO (zkCallback-2528-thread-4-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (7) -> (10)
[junit4] 2> 1600676 INFO (zkCallback-2536-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (10) -> (11)
[junit4] 2> 1600677 INFO (zkCallback-2528-thread-4-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (10) -> (11)
[junit4] 2> 1600677 INFO (zkCallback-2536-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (11) -> (12)
[junit4] 2> 1600678 INFO (zkCallback-2528-thread-4-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (11) -> (12)
[junit4] 2> 1600679 INFO (zkCallback-2528-thread-4-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (12) -> (14)
[junit4] 2> 1600679 INFO (zkCallback-2536-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (12) -> (14)
[junit4] 2> 1600680 INFO (zkCallback-2528-thread-4-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (14) -> (16)
[junit4] 2> 1600680 INFO (zkCallback-2536-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (14) -> (16)
[junit4] 2> 1600681 INFO (zkCallback-2536-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (16) -> (17)
[junit4] 2> 1600681 INFO (zkCallback-2528-thread-4-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (16) -> (17)
[junit4] 2> 1600682 INFO (zkCallback-2528-thread-4-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (17) -> (18)
[junit4] 2> 1600682 INFO (zkCallback-2536-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (17) -> (18)
[junit4] 2> 1600683 INFO (zkCallback-2536-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (18) -> (20)
[junit4] 2> 1600683 INFO (zkCallback-2528-thread-4-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (18) -> (20)
[junit4] 2> 1600684 INFO (zkCallback-2536-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (20) -> (21)
[junit4] 2> 1600684 INFO (zkCallback-2528-thread-4-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (20) -> (21)
[junit4] 2> 1600686 INFO (zkConnectionManagerCallback-2649-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600688 INFO (zkCallback-2528-thread-4-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (21) -> (17)
[junit4] 2> 1600688 INFO (zkCallback-2536-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (21) -> (17)
[junit4] 2> 1600689 INFO (zkCallback-2528-thread-2-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (17) -> (13)
[junit4] 2> 1600689 INFO (zkCallback-2536-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (17) -> (13)
[junit4] 2> 1600690 INFO (zkCallback-2528-thread-1-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (13) -> (9)
[junit4] 2> 1600690 INFO (zkCallback-2536-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (13) -> (9)
[junit4] 2> 1600691 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x104c7639a33003b, likely client has closed socket
[junit4] 2> 1600691 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (9) -> (5)
[junit4] 2> 1600691 INFO (zkCallback-2528-thread-3-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (9) -> (5)
[junit4] 2> 1600692 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (1)
[junit4] 2> 1600693 INFO (zkCallback-2528-thread-5-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (1)
[junit4] 2> 1600694 INFO (zkConnectionManagerCallback-2651-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600696 INFO (TEST-TestStressLiveNodes.testStress-seed#[D11AA6B8CC8BF8CD]) [ ] o.a.s.c.TestStressLiveNodes preparing parallel adds to live nodes: iter=11, numThreads=2 numNodesPerThread=2
[junit4] 2> 1600697 INFO (zkConnectionManagerCallback-2653-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600698 INFO (zkConnectionManagerCallback-2655-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600700 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1600700 INFO (zkCallback-2528-thread-5-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1600700 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1600701 INFO (zkCallback-2528-thread-5-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1600701 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1600702 INFO (zkCallback-2528-thread-5-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1600702 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 1600703 INFO (zkCallback-2528-thread-5-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 1600704 INFO (zkConnectionManagerCallback-2657-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600706 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (3)
[junit4] 2> 1600706 INFO (zkCallback-2528-thread-5-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (3)
[junit4] 2> 1600709 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x104c7639a330040, likely client has closed socket
[junit4] 2> 1600710 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (1)
[junit4] 2> 1600710 INFO (zkCallback-2528-thread-5-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (1)
[junit4] 2> 1600711 INFO (zkConnectionManagerCallback-2659-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600712 INFO (TEST-TestStressLiveNodes.testStress-seed#[D11AA6B8CC8BF8CD]) [ ] o.a.s.c.TestStressLiveNodes preparing parallel adds to live nodes: iter=12, numThreads=4 numNodesPerThread=2
[junit4] 2> 1600713 INFO (zkConnectionManagerCallback-2661-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600715 INFO (zkConnectionManagerCallback-2663-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600716 INFO (zkConnectionManagerCallback-2665-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600718 INFO (zkConnectionManagerCallback-2667-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600720 INFO (zkCallback-2528-thread-5-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1600720 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1600721 INFO (zkCallback-2528-thread-5-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (5)
[junit4] 2> 1600721 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (5)
[junit4] 2> 1600722 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (6)
[junit4] 2> 1600722 INFO (zkCallback-2528-thread-5-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (6)
[junit4] 2> 1600723 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (6) -> (8)
[junit4] 2> 1600723 INFO (zkCallback-2528-thread-5-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (6) -> (9)
[junit4] 2> 1600723 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (8) -> (9)
[junit4] 2> 1600725 INFO (zkConnectionManagerCallback-2669-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600726 INFO (zkCallback-2528-thread-5-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (9) -> (7)
[junit4] 2> 1600726 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (9) -> (7)
[junit4] 2> 1600727 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (7) -> (5)
[junit4] 2> 1600728 INFO (zkCallback-2528-thread-3-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (7) -> (5)
[junit4] 2> 1600728 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x104c7639a330045, likely client has closed socket
[junit4] 2> 1600728 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (3)
[junit4] 2> 1600728 INFO (zkCallback-2528-thread-1-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (3)
[junit4] 2> 1600729 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (1)
[junit4] 2> 1600729 INFO (zkCallback-2528-thread-5-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (1)
[junit4] 2> 1600730 INFO (zkConnectionManagerCallback-2671-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600732 INFO (TEST-TestStressLiveNodes.testStress-seed#[D11AA6B8CC8BF8CD]) [ ] o.a.s.c.TestStressLiveNodes preparing parallel adds to live nodes: iter=13, numThreads=4 numNodesPerThread=5
[junit4] 2> 1600733 INFO (zkConnectionManagerCallback-2673-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600734 INFO (zkConnectionManagerCallback-2675-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600736 INFO (zkConnectionManagerCallback-2677-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600737 INFO (zkConnectionManagerCallback-2679-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600740 INFO (zkCallback-2528-thread-5-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1600740 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1600740 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1600740 INFO (zkCallback-2528-thread-5-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1600742 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (5)
[junit4] 2> 1600742 INFO (zkCallback-2528-thread-5-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (5)
[junit4] 2> 1600743 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (7)
[junit4] 2> 1600743 INFO (zkCallback-2528-thread-1-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (7)
[junit4] 2> 1600750 INFO (zkCallback-2528-thread-1-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (7) -> (8)
[junit4] 2> 1600750 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (7) -> (8)
[junit4] 2> 1600750 INFO (zkCallback-2536-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (8) -> (9)
[junit4] 2> 1600750 INFO (zkCallback-2528-thread-5-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (8) -> (9)
[junit4] 2> 1600751 INFO (zkCallback-2536-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (9) -> (11)
[junit4] 2> 1600752 INFO (zkCallback-2528-thread-5-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (9) -> (11)
[junit4] 2> 1600752 INFO (zkCallback-2536-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (11) -> (12)
[junit4] 2> 1600752 INFO (zkCallback-2528-thread-1-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (11) -> (12)
[junit4] 2> 1600754 INFO (zkCallback-2528-thread-1-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (12) -> (13)
[junit4] 2> 1600754 INFO (zkCallback-2536-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (12) -> (13)
[junit4] 2> 1600754 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (13) -> (14)
[junit4] 2> 1600755 INFO (zkCallback-2528-thread-1-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (13) -> (15)
[junit4] 2> 1600755 INFO (zkCallback-2536-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (14) -> (15)
[junit4] 2> 1600756 INFO (zkCallback-2528-thread-1-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (15) -> (16)
[junit4] 2> 1600756 INFO (zkCallback-2536-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (15) -> (16)
[junit4] 2> 1600758 INFO (zkCallback-2536-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (16) -> (17)
[junit4] 2> 1600758 INFO (zkCallback-2528-thread-1-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (16) -> (17)
[junit4] 2> 1600759 INFO (zkCallback-2528-thread-5-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (17) -> (19)
[junit4] 2> 1600759 INFO (zkCallback-2536-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (17) -> (19)
[junit4] 2> 1600761 INFO (zkCallback-2528-thread-5-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (19) -> (21)
[junit4] 2> 1600761 INFO (zkCallback-2536-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (19) -> (21)
[junit4] 2> 1600769 INFO (zkConnectionManagerCallback-2681-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600772 INFO (zkCallback-2528-thread-5-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (21) -> (16)
[junit4] 2> 1600772 INFO (zkCallback-2536-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (21) -> (11)
[junit4] 2> 1600772 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x104c7639a33004a, likely client has closed socket
[junit4] 2> 1600773 INFO (zkCallback-2528-thread-1-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (16) -> (6)
[junit4] 2> 1600774 INFO (zkCallback-2536-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (11) -> (1)
[junit4] 2> 1600774 INFO (zkCallback-2528-thread-3-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (6) -> (1)
[junit4] 2> 1600777 INFO (zkConnectionManagerCallback-2683-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600779 INFO (TEST-TestStressLiveNodes.testStress-seed#[D11AA6B8CC8BF8CD]) [ ] o.a.s.c.TestStressLiveNodes preparing parallel adds to live nodes: iter=14, numThreads=5 numNodesPerThread=1
[junit4] 2> 1600780 INFO (zkConnectionManagerCallback-2685-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600782 INFO (zkConnectionManagerCallback-2687-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600783 INFO (zkConnectionManagerCallback-2689-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600785 INFO (zkConnectionManagerCallback-2691-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600786 INFO (zkConnectionManagerCallback-2693-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600789 INFO (zkCallback-2528-thread-1-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1600789 INFO (zkCallback-2536-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1600789 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (4)
[junit4] 2> 1600789 INFO (zkCallback-2528-thread-1-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (4)
[junit4] 2> 1600791 INFO (zkCallback-2528-thread-1-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 1600793 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 1600800 INFO (zkCallback-2528-thread-1-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (6)
[junit4] 2> 1600800 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (6)
[junit4] 2> 1600801 INFO (zkConnectionManagerCallback-2695-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600803 INFO (zkCallback-2528-thread-1-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (6) -> (5)
[junit4] 2> 1600803 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (6) -> (5)
[junit4] 2> 1600805 INFO (zkCallback-2528-thread-3-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (3)
[junit4] 2> 1600805 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (3)
[junit4] 2> 1600805 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x104c7639a330052, likely client has closed socket
[junit4] 2> 1600806 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (1)
[junit4] 2> 1600806 INFO (zkCallback-2528-thread-1-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (1)
[junit4] 2> 1600807 INFO (zkConnectionManagerCallback-2697-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600809 INFO (TEST-TestStressLiveNodes.testStress-seed#[D11AA6B8CC8BF8CD]) [ ] o.a.s.c.TestStressLiveNodes preparing parallel adds to live nodes: iter=15, numThreads=4 numNodesPerThread=2
[junit4] 2> 1600810 INFO (zkConnectionManagerCallback-2699-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600811 INFO (zkConnectionManagerCallback-2701-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600813 INFO (zkConnectionManagerCallback-2703-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600815 INFO (zkConnectionManagerCallback-2705-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1600817 INFO (zkCallback-2536-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1600817 INFO (zkCallback-2528-thread-1-processing-n:127.0.0.1:42398_solr) [n:127.0.0.1:42398_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1
[...truncated too long message...]
lrClient.java:1104)
[junit4] > at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:884)
[junit4] > at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:817)
[junit4] > at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
[junit4] > at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
[junit4] > at org.apache.solr.cloud.UnloadDistributedZkTest.testUnloadShardAndCollection(UnloadDistributedZkTest.java:125)
[junit4] > at org.apache.solr.cloud.UnloadDistributedZkTest.test(UnloadDistributedZkTest.java:72)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 4027818 WARN (SUITE-HdfsUnloadDistributedZkTest-seed#[D11AA6B8CC8BF8CD]-worker) [ ] o.a.h.h.s.d.DirectoryScanner DirectoryScanner: shutdown has been called
[junit4] 2> 4027836 INFO (SUITE-HdfsUnloadDistributedZkTest-seed#[D11AA6B8CC8BF8CD]-worker) [ ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0
[junit4] 2> 4027940 WARN (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_D11AA6B8CC8BF8CD-001/tempDir-001/hdfsBaseDir/data/data3/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_D11AA6B8CC8BF8CD-001/tempDir-001/hdfsBaseDir/data/data4/]] heartbeating to localhost/127.0.0.1:35393) [ ] o.a.h.h.s.d.IncrementalBlockReportManager IncrementalBlockReportManager interrupted
[junit4] 2> 4027940 WARN (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_D11AA6B8CC8BF8CD-001/tempDir-001/hdfsBaseDir/data/data3/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_D11AA6B8CC8BF8CD-001/tempDir-001/hdfsBaseDir/data/data4/]] heartbeating to localhost/127.0.0.1:35393) [ ] o.a.h.h.s.d.DataNode Ending block pool service for: Block pool BP-151876515-10.41.0.5-1520275518190 (Datanode Uuid 9fa542e3-0cc4-48ea-a8ce-d20d95a52b4d) service to localhost/127.0.0.1:35393
[junit4] 2> 4027945 WARN (SUITE-HdfsUnloadDistributedZkTest-seed#[D11AA6B8CC8BF8CD]-worker) [ ] o.a.h.h.s.d.DirectoryScanner DirectoryScanner: shutdown has been called
[junit4] 2> 4027979 INFO (SUITE-HdfsUnloadDistributedZkTest-seed#[D11AA6B8CC8BF8CD]-worker) [ ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0
[junit4] 2> 4028081 WARN (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_D11AA6B8CC8BF8CD-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_D11AA6B8CC8BF8CD-001/tempDir-001/hdfsBaseDir/data/data2/]] heartbeating to localhost/127.0.0.1:35393) [ ] o.a.h.h.s.d.IncrementalBlockReportManager IncrementalBlockReportManager interrupted
[junit4] 2> 4028081 WARN (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_D11AA6B8CC8BF8CD-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_D11AA6B8CC8BF8CD-001/tempDir-001/hdfsBaseDir/data/data2/]] heartbeating to localhost/127.0.0.1:35393) [ ] o.a.h.h.s.d.DataNode Ending block pool service for: Block pool BP-151876515-10.41.0.5-1520275518190 (Datanode Uuid aaed515d-744c-46d8-b4a4-171cf31d5895) service to localhost/127.0.0.1:35393
[junit4] 2> 4028096 INFO (SUITE-HdfsUnloadDistributedZkTest-seed#[D11AA6B8CC8BF8CD]-worker) [ ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_D11AA6B8CC8BF8CD-001
[junit4] 2> Mar 05, 2018 6:46:28 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 34 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {other_tl1=PostingsFormat(name=Asserting), range_facet_l_dv=Lucene50(blocksize=128), multiDefault=PostingsFormat(name=Memory), a_t=PostingsFormat(name=Memory), intDefault=PostingsFormat(name=Asserting), id_i1=PostingsFormat(name=Asserting), a_i1=PostingsFormat(name=Memory), id=Lucene50(blocksize=128), range_facet_i_dv=PostingsFormat(name=Memory), text=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), range_facet_l=PostingsFormat(name=Memory), timestamp=PostingsFormat(name=Memory)}, docValues:{other_tl1=DocValuesFormat(name=Lucene70), range_facet_l_dv=DocValuesFormat(name=Direct), _version_=DocValuesFormat(name=Lucene70), intDefault=DocValuesFormat(name=Lucene70), id_i1=DocValuesFormat(name=Lucene70), a_i1=DocValuesFormat(name=Lucene70), range_facet_i_dv=DocValuesFormat(name=Lucene70), intDvoDefault=DocValuesFormat(name=Direct), range_facet_l=DocValuesFormat(name=Lucene70), timestamp=DocValuesFormat(name=Lucene70)}, maxPointsInLeafNode=508, maxMBSortInHeap=7.46447025951051, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@10a078ed), locale=es-MX, timezone=Asia/Ulan_Bator
[junit4] 2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=2,free=232380352,total=523763712
[junit4] 2> NOTE: All tests run in this JVM: [LargeFieldTest, TestFuzzyAnalyzedSuggestions, TestPivotHelperCode, AnalyticsQueryTest, QueryEqualityTest, SuggesterFSTTest, TestFieldResource, TestSolrCloudWithSecureImpersonation, TestFunctionQuery, FastVectorHighlighterTest, TestUseDocValuesAsStored, TestReplicationHandler, SpellCheckCollatorWithCollapseTest, LeaderInitiatedRecoveryOnShardRestartTest, TestHdfsUpdateLog, SearchHandlerTest, TestFileDictionaryLookup, SolrCoreTest, HttpPartitionOnCommitTest, TestReqParamsAPI, TestDistributedStatsComponentCardinality, TestSolrFieldCacheBean, DeleteStatusTest, SliceStateTest, TestSurroundQueryParser, PrimUtilsTest, TestCollectionsAPIViaSolrCloudCluster, DistributedMLTComponentTest, TestNodeLostTrigger, TestNumericRangeQuery64, HdfsChaosMonkeySafeLeaderTest, TestXIncludeConfig, OverseerTaskQueueTest, TestSolrJ, InfixSuggestersTest, JsonLoaderTest, UpdateLogTest, TestTriggerIntegration, QueryElevationComponentTest, TestSearchPerf, EnumFieldTest, TestElisionMultitermQuery, RemoteQueryErrorTest, TestSolrCoreSnapshots, TestUninvertingReader, MissingSegmentRecoveryTest, PreAnalyzedUpdateProcessorTest, SyncSliceTest, AddReplicaTest, DistributedFacetSimpleRefinementLongTailTest, TestRebalanceLeaders, TestJavabinTupleStreamParser, TestPolicyCloud, SpatialHeatmapFacetsTest, HdfsThreadLeakTest, IndexSchemaTest, XsltUpdateRequestHandlerTest, TestRestManager, TestCharFilters, ShardSplitTest, CurrencyFieldTypeTest, PeerSyncTest, TestSchemaVersionResource, DeleteNodeTest, TestPayloadScoreQParserPlugin, SignatureUpdateProcessorFactoryTest, TestSolrCLIRunExample, TestExclusionRuleCollectionAccess, TestJsonRequest, TestCollationFieldDocValues, TestFieldSortValues, SampleTest, TestDistributedGrouping, TestPullReplicaErrorHandling, TestGenericDistributedQueue, TestClusterProperties, SimplePostToolTest, TestBulkSchemaConcurrent, SOLR749Test, TestDirectoryFactory, CollectionStateFormat2Test, SortByFunctionTest, PKIAuthenticationIntegrationTest, TestReversedWildcardFilterFactory, DocumentBuilderTest, BitVectorTest, TestReplicaProperties, TestLegacyFieldCache, LoggingHandlerTest, TestLegacyField, TestMinMaxOnMultiValuedField, TestReload, PrimitiveFieldTypeTest, ShardRoutingCustomTest, UninvertDocValuesMergePolicyTest, DistributedDebugComponentTest, FileUtilsTest, TestManagedStopFilterFactory, AsyncCallRequestStatusResponseTest, SolrShardReporterTest, OverseerTest, ScheduledTriggerTest, TestPartialUpdateDeduplication, TestSchemaManager, BlockJoinFacetSimpleTest, HdfsBasicDistributedZk2Test, SolrInfoBeanTest, ScriptEngineTest, TestStressLiveNodes, RequiredFieldsTest, SchemaApiFailureTest, TestStressRecovery, TestSmileRequest, ChaosMonkeySafeLeaderTest, TestDistribIDF, HdfsWriteToMultipleCollectionsTest, BasicFunctionalityTest, TestOnReconnectListenerSupport, TestSubQueryTransformerDistrib, HdfsRecoveryZkTest, TestStressVersions, SortSpecParsingTest, SystemInfoHandlerTest, PolyFieldTest, HLLSerializationTest, LeaderElectionContextKeyTest, TestPostingsSolrHighlighter, BasicDistributedZkTest, SimpleCollectionCreateDeleteTest, TestLocalFSCloudBackupRestore, HdfsAutoAddReplicasIntegrationTest, MetricTriggerTest, SearchRateTriggerTest, TestComputePlanAction, TestExecutePlanAction, TestNodeAddedTrigger, TestSimDistributedQueue, CdcrReplicationDistributedZkTest, CdcrRequestHandlerTest, HdfsNNFailoverTest, HdfsRestartWhileUpdatingTest, HdfsUnloadDistributedZkTest]
[junit4] Completed [600/780 (3!)] on J1 in 79.99s, 1 test, 1 error <<< FAILURES!
[...truncated 50019 lines...]
[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1494 - Still
Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1494/
4 tests failed.
FAILED: org.apache.solr.cloud.cdcr.CdcrReplicationHandlerTest.testReplicationWithBufferedUpdates
Error Message:
Timeout while trying to assert number of documents @ https://127.0.0.1:50859/rqgf/zx/source_collection_shard1_replica_n1/
Stack Trace:
java.lang.AssertionError: Timeout while trying to assert number of documents @ https://127.0.0.1:50859/rqgf/zx/source_collection_shard1_replica_n1/
at __randomizedtesting.SeedInfo.seed([BBB8DF46830C4269:68B18F58C69FDEFE]:0)
at org.apache.solr.cloud.cdcr.CdcrReplicationHandlerTest.assertNumDocs(CdcrReplicationHandlerTest.java:257)
at org.apache.solr.cloud.cdcr.CdcrReplicationHandlerTest.testReplicationWithBufferedUpdates(CdcrReplicationHandlerTest.java:237)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.AssertionError: expected:<0> but was:<10>
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:472)
at org.junit.Assert.assertEquals(Assert.java:456)
at org.apache.solr.cloud.cdcr.CdcrReplicationHandlerTest.assertNumDocs(CdcrReplicationHandlerTest.java:248)
... 42 more
FAILED: org.apache.solr.cloud.MoveReplicaHDFSTest.testFailedMove
Error Message:
Stack Trace:
java.lang.AssertionError
at __randomizedtesting.SeedInfo.seed([BBB8DF46830C4269:11750CB434DF97B9]:0)
at org.junit.Assert.fail(Assert.java:92)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertFalse(Assert.java:68)
at org.junit.Assert.assertFalse(Assert.java:79)
at org.apache.solr.cloud.MoveReplicaTest.testFailedMove(MoveReplicaTest.java:303)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.MoveReplicaHDFSTest.test
Error Message:
Error from server at http://127.0.0.1:43657/solr: delete the collection time out:180s
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:43657/solr: delete the collection time out:180s
at __randomizedtesting.SeedInfo.seed([BBB8DF46830C4269:33ECE09C2DF02F91]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1104)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:884)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:817)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
at org.apache.solr.cloud.MiniSolrCloudCluster.deleteAllCollections(MiniSolrCloudCluster.java:451)
at org.apache.solr.cloud.MoveReplicaTest.beforeTest(MoveReplicaTest.java:97)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:968)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.MoveReplicaHDFSTest.testNormalMove
Error Message:
Error from server at http://127.0.0.1:43657/solr: delete the collection time out:180s
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:43657/solr: delete the collection time out:180s
at __randomizedtesting.SeedInfo.seed([BBB8DF46830C4269:1D6068E106ED8073]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1104)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:884)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:817)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
at org.apache.solr.cloud.MiniSolrCloudCluster.deleteAllCollections(MiniSolrCloudCluster.java:451)
at org.apache.solr.cloud.MoveReplicaTest.beforeTest(MoveReplicaTest.java:97)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:968)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 14384 lines...]
[junit4] Suite: org.apache.solr.cloud.MoveReplicaHDFSTest
[junit4] 2> 3296612 INFO (SUITE-MoveReplicaHDFSTest-seed#[BBB8DF46830C4269]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.MoveReplicaHDFSTest_BBB8DF46830C4269-001/init-core-data-001
[junit4] 2> 3296613 WARN (SUITE-MoveReplicaHDFSTest-seed#[BBB8DF46830C4269]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=33 numCloses=33
[junit4] 2> 3296613 INFO (SUITE-MoveReplicaHDFSTest-seed#[BBB8DF46830C4269]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 3296615 INFO (SUITE-MoveReplicaHDFSTest-seed#[BBB8DF46830C4269]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 3296615 INFO (SUITE-MoveReplicaHDFSTest-seed#[BBB8DF46830C4269]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 4 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.MoveReplicaHDFSTest_BBB8DF46830C4269-001/tempDir-001
[junit4] 2> 3296616 INFO (SUITE-MoveReplicaHDFSTest-seed#[BBB8DF46830C4269]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 3296616 INFO (Thread-10757) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 3296616 INFO (Thread-10757) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 3296618 ERROR (Thread-10757) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 3296716 INFO (SUITE-MoveReplicaHDFSTest-seed#[BBB8DF46830C4269]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:40718
[junit4] 2> 3296718 INFO (zkConnectionManagerCallback-52590-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3296721 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x104c28562f50000, likely client has closed socket
[junit4] 2> 3296723 INFO (jetty-launcher-52587-thread-1) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T22:27:37+01:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 3296723 INFO (jetty-launcher-52587-thread-2) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T22:27:37+01:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 3296724 INFO (jetty-launcher-52587-thread-4) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T22:27:37+01:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 3296724 INFO (jetty-launcher-52587-thread-3) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T22:27:37+01:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 3296725 INFO (jetty-launcher-52587-thread-1) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 3296725 INFO (jetty-launcher-52587-thread-1) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 3296725 INFO (jetty-launcher-52587-thread-1) [ ] o.e.j.s.session Scavenging every 600000ms
[junit4] 2> 3296726 INFO (jetty-launcher-52587-thread-2) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 3296726 INFO (jetty-launcher-52587-thread-2) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 3296726 INFO (jetty-launcher-52587-thread-2) [ ] o.e.j.s.session Scavenging every 660000ms
[junit4] 2> 3296726 INFO (jetty-launcher-52587-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6fa51df5{/solr,null,AVAILABLE}
[junit4] 2> 3296727 INFO (jetty-launcher-52587-thread-3) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 3296727 INFO (jetty-launcher-52587-thread-3) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 3296727 INFO (jetty-launcher-52587-thread-3) [ ] o.e.j.s.session Scavenging every 600000ms
[junit4] 2> 3296727 INFO (jetty-launcher-52587-thread-3) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@43cb93f{/solr,null,AVAILABLE}
[junit4] 2> 3296728 INFO (jetty-launcher-52587-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@57f6781a{/solr,null,AVAILABLE}
[junit4] 2> 3296728 INFO (jetty-launcher-52587-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@2e0a720b{HTTP/1.1,[http/1.1]}{127.0.0.1:43657}
[junit4] 2> 3296728 INFO (jetty-launcher-52587-thread-1) [ ] o.e.j.s.Server Started @3299837ms
[junit4] 2> 3296728 INFO (jetty-launcher-52587-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=43657}
[junit4] 2> 3296729 ERROR (jetty-launcher-52587-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3296729 INFO (jetty-launcher-52587-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 8.0.0
[junit4] 2> 3296729 INFO (jetty-launcher-52587-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3296729 INFO (jetty-launcher-52587-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3296729 INFO (jetty-launcher-52587-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-03-04T19:24:46.685Z
[junit4] 2> 3296730 INFO (jetty-launcher-52587-thread-4) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 3296730 INFO (jetty-launcher-52587-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@32f527ae{HTTP/1.1,[http/1.1]}{127.0.0.1:41853}
[junit4] 2> 3296730 INFO (jetty-launcher-52587-thread-4) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 3296730 INFO (jetty-launcher-52587-thread-2) [ ] o.e.j.s.Server Started @3299838ms
[junit4] 2> 3296730 INFO (jetty-launcher-52587-thread-4) [ ] o.e.j.s.session Scavenging every 660000ms
[junit4] 2> 3296730 INFO (jetty-launcher-52587-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=41853}
[junit4] 2> 3296730 ERROR (jetty-launcher-52587-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3296730 INFO (jetty-launcher-52587-thread-4) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@a3b46c7{/solr,null,AVAILABLE}
[junit4] 2> 3296730 INFO (jetty-launcher-52587-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 8.0.0
[junit4] 2> 3296730 INFO (jetty-launcher-52587-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3296730 INFO (jetty-launcher-52587-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3296731 INFO (jetty-launcher-52587-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-03-04T19:24:46.686Z
[junit4] 2> 3296731 INFO (jetty-launcher-52587-thread-4) [ ] o.e.j.s.AbstractConnector Started ServerConnector@64f8c16{HTTP/1.1,[http/1.1]}{127.0.0.1:48734}
[junit4] 2> 3296731 INFO (jetty-launcher-52587-thread-3) [ ] o.e.j.s.AbstractConnector Started ServerConnector@7142cce9{HTTP/1.1,[http/1.1]}{127.0.0.1:55105}
[junit4] 2> 3296731 INFO (jetty-launcher-52587-thread-3) [ ] o.e.j.s.Server Started @3299840ms
[junit4] 2> 3296731 INFO (jetty-launcher-52587-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=55105}
[junit4] 2> 3296732 ERROR (jetty-launcher-52587-thread-3) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3296732 INFO (jetty-launcher-52587-thread-3) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 8.0.0
[junit4] 2> 3296732 INFO (jetty-launcher-52587-thread-3) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3296732 INFO (jetty-launcher-52587-thread-3) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3296732 INFO (jetty-launcher-52587-thread-3) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-03-04T19:24:46.688Z
[junit4] 2> 3296732 INFO (zkConnectionManagerCallback-52594-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3296732 INFO (jetty-launcher-52587-thread-4) [ ] o.e.j.s.Server Started @3299841ms
[junit4] 2> 3296732 INFO (jetty-launcher-52587-thread-4) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=48734}
[junit4] 2> 3296733 ERROR (jetty-launcher-52587-thread-4) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3296733 INFO (jetty-launcher-52587-thread-4) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 8.0.0
[junit4] 2> 3296733 INFO (jetty-launcher-52587-thread-4) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3296733 INFO (jetty-launcher-52587-thread-4) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3296733 INFO (jetty-launcher-52587-thread-4) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-03-04T19:24:46.689Z
[junit4] 2> 3296733 INFO (jetty-launcher-52587-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 3296740 INFO (zkConnectionManagerCallback-52596-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3296741 INFO (zkConnectionManagerCallback-52592-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3296741 INFO (jetty-launcher-52587-thread-3) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 3296743 INFO (jetty-launcher-52587-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 3296743 INFO (zkConnectionManagerCallback-52598-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3296744 INFO (jetty-launcher-52587-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40718/solr
[junit4] 2> 3296749 INFO (jetty-launcher-52587-thread-4) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 3296751 INFO (zkConnectionManagerCallback-52602-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3296752 INFO (jetty-launcher-52587-thread-3) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40718/solr
[junit4] 2> 3296754 INFO (jetty-launcher-52587-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40718/solr
[junit4] 2> 3296754 DEBUG (jetty-launcher-52587-thread-2) [n:127.0.0.1:41853_solr ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$263/752094002@34224795
[junit4] 2> 3296755 INFO (zkConnectionManagerCallback-52607-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3296756 DEBUG (jetty-launcher-52587-thread-3) [n:127.0.0.1:55105_solr ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$263/752094002@79b29abd
[junit4] 2> 3296756 INFO (zkConnectionManagerCallback-52610-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3296757 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x104c28562f50007, likely client has closed socket
[junit4] 2> 3296757 DEBUG (jetty-launcher-52587-thread-1) [n:127.0.0.1:43657_solr ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$263/752094002@4e6b81b
[junit4] 2> 3296757 INFO (jetty-launcher-52587-thread-4) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40718/solr
[junit4] 2> 3296759 INFO (zkConnectionManagerCallback-52614-thread-1-processing-n:127.0.0.1:55105_solr) [n:127.0.0.1:55105_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3296759 INFO (zkConnectionManagerCallback-52618-thread-1-processing-n:127.0.0.1:43657_solr) [n:127.0.0.1:43657_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3296760 INFO (zkConnectionManagerCallback-52612-thread-1-processing-n:127.0.0.1:41853_solr) [n:127.0.0.1:41853_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3296760 INFO (zkConnectionManagerCallback-52620-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3296762 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x104c28562f5000a, likely client has closed socket
[junit4] 2> 3296764 DEBUG (jetty-launcher-52587-thread-4) [n:127.0.0.1:48734_solr ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$263/752094002@7ae6dc45
[junit4] 2> 3296780 INFO (zkConnectionManagerCallback-52622-thread-1-processing-n:127.0.0.1:48734_solr) [n:127.0.0.1:48734_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3296906 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/clusterprops.json
[junit4] 2> 3296906 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 3296906 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterstate.json
[junit4] 2> 3296907 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 3296907 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/aliases.json
[junit4] 2> 3296907 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/clusterprops.json
[junit4] 2> 3296908 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/security.json
[junit4] 2> 3296908 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 3296908 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterstate.json
[junit4] 2> 3296909 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 3296909 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/aliases.json
[junit4] 2> 3296910 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/security.json
[junit4] 2> 3296912 INFO (jetty-launcher-52587-thread-4) [n:127.0.0.1:48734_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 3296912 DEBUG (jetty-launcher-52587-thread-4) [n:127.0.0.1:48734_solr ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/73397372091432972-127.0.0.1:48734_solr-n_0000000000
[junit4] 2> 3296912 INFO (jetty-launcher-52587-thread-3) [n:127.0.0.1:55105_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 3296913 DEBUG (jetty-launcher-52587-thread-3) [n:127.0.0.1:55105_solr ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/73397372091432968-127.0.0.1:55105_solr-n_0000000001
[junit4] 2> 3296913 INFO (jetty-launcher-52587-thread-4) [n:127.0.0.1:48734_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:48734_solr
[junit4] 2> 3296913 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/overseer_elect/election/73397372091432972-127.0.0.1:48734_solr-n_0000000000
[junit4] 2> 3296913 DEBUG (jetty-launcher-52587-thread-3) [n:127.0.0.1:55105_solr ] o.a.s.c.LeaderElector Watching path /overseer_elect/election/73397372091432972-127.0.0.1:48734_solr-n_0000000000 to know if I could be the leader
[junit4] 2> 3296914 INFO (jetty-launcher-52587-thread-3) [n:127.0.0.1:55105_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:55105_solr
[junit4] 2> 3296914 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 3296915 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 3296919 INFO (jetty-launcher-52587-thread-4) [n:127.0.0.1:48734_solr ] o.a.s.c.Overseer Overseer (id=73397372091432972-127.0.0.1:48734_solr-n_0000000000) starting
[junit4] 2> 3296920 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 3296920 INFO (zkCallback-52613-thread-1-processing-n:127.0.0.1:55105_solr) [n:127.0.0.1:55105_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3296920 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 3296921 INFO (zkCallback-52621-thread-1-processing-n:127.0.0.1:48734_solr) [n:127.0.0.1:48734_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3296925 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/clusterprops.json
[junit4] 2> 3296925 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 3296926 INFO (jetty-launcher-52587-thread-1) [n:127.0.0.1:43657_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3296926 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterstate.json
[junit4] 2> 3296926 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 3296927 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/aliases.json
[junit4] 2> 3296927 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/clusterprops.json
[junit4] 2> 3296928 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 3296928 INFO (jetty-launcher-52587-thread-2) [n:127.0.0.1:41853_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3296928 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/security.json
[junit4] 2> 3296928 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterstate.json
[junit4] 2> 3296929 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 3296929 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/aliases.json
[junit4] 2> 3296930 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/security.json
[junit4] 2> 3296930 INFO (jetty-launcher-52587-thread-1) [n:127.0.0.1:43657_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 3296931 DEBUG (jetty-launcher-52587-thread-1) [n:127.0.0.1:43657_solr ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/73397372091432969-127.0.0.1:43657_solr-n_0000000002
[junit4] 2> 3296931 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/overseer_elect/election/73397372091432968-127.0.0.1:55105_solr-n_0000000001
[junit4] 2> 3296931 DEBUG (jetty-launcher-52587-thread-1) [n:127.0.0.1:43657_solr ] o.a.s.c.LeaderElector Watching path /overseer_elect/election/73397372091432968-127.0.0.1:55105_solr-n_0000000001 to know if I could be the leader
[junit4] 2> 3296932 INFO (jetty-launcher-52587-thread-2) [n:127.0.0.1:41853_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 3296932 DEBUG (jetty-launcher-52587-thread-1) [n:127.0.0.1:43657_solr ] o.a.s.c.ZkController Publish node=127.0.0.1:43657_solr as DOWN
[junit4] 2> 3296933 DEBUG (jetty-launcher-52587-thread-2) [n:127.0.0.1:41853_solr ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/73397372091432971-127.0.0.1:41853_solr-n_0000000003
[junit4] 2> 3296933 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/overseer_elect/election/73397372091432969-127.0.0.1:43657_solr-n_0000000002
[junit4] 2> 3296934 DEBUG (OverseerCollectionConfigSetProcessor-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.OverseerTaskProcessor Process current queue of overseer operations
[junit4] 2> 3296934 INFO (jetty-launcher-52587-thread-1) [n:127.0.0.1:43657_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43657_solr
[junit4] 2> 3296934 DEBUG (jetty-launcher-52587-thread-2) [n:127.0.0.1:41853_solr ] o.a.s.c.LeaderElector Watching path /overseer_elect/election/73397372091432969-127.0.0.1:43657_solr-n_0000000002 to know if I could be the leader
[junit4] 2> 3296934 DEBUG (jetty-launcher-52587-thread-2) [n:127.0.0.1:41853_solr ] o.a.s.c.ZkController Publish node=127.0.0.1:41853_solr as DOWN
[junit4] 2> 3296934 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 3296934 DEBUG (OverseerStateUpdate-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.Overseer Starting to work on the main queue
[junit4] 2> 3296935 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 3296935 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 3296935 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 3296936 DEBUG (OverseerCollectionConfigSetProcessor-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
[junit4] 2> 3296936 DEBUG (OverseerCollectionConfigSetProcessor-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
[junit4] 2> 3296936 DEBUG (OverseerCollectionConfigSetProcessor-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
[junit4] 2> 3296936 DEBUG (OverseerCollectionConfigSetProcessor-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
[junit4] 2> 3296936 DEBUG (OverseerCollectionConfigSetProcessor-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
[junit4] 2> 3296936 DEBUG (OverseerCollectionConfigSetProcessor-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
[junit4] 2> 3296936 INFO (OverseerStateUpdate-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3296936 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/collection-queue-work
[junit4] 2> 3296937 DEBUG (jetty-launcher-52587-thread-4) [n:127.0.0.1:48734_solr ] o.a.s.c.ZkController Publish node=127.0.0.1:48734_solr as DOWN
[junit4] 2> 3296937 INFO (jetty-launcher-52587-thread-2) [n:127.0.0.1:41853_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41853_solr
[junit4] 2> 3296937 DEBUG (OverseerAutoScalingTriggerThread-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
[junit4] 2> 3296938 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue-work
[junit4] 2> 3296942 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
[junit4] 2> 3296942 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 3296942 INFO (zkCallback-52621-thread-1-processing-n:127.0.0.1:48734_solr) [n:127.0.0.1:48734_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 3296943 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/autoscaling.json
[junit4] 2> 3296943 DEBUG (OverseerAutoScalingTriggerThread-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
[junit4] 2> 3296944 DEBUG (OverseerStateUpdate-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.Overseer processMessage: queueSize: 2, message = {
[junit4] 2> "operation":"downnode",
[junit4] 2> "node_name":"127.0.0.1:43657_solr"} current state version: 0
[junit4] 2> 3296944 DEBUG (OverseerStateUpdate-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.o.NodeMutator DownNode state invoked for node: 127.0.0.1:43657_solr
[junit4] 2> 3296944 DEBUG (OverseerStateUpdate-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.Overseer processMessage: queueSize: 2, message = {
[junit4] 2> "operation":"downnode",
[junit4] 2> "node_name":"127.0.0.1:41853_solr"} current state version: 0
[junit4] 2> 3296944 DEBUG (OverseerStateUpdate-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.o.NodeMutator DownNode state invoked for node: 127.0.0.1:41853_solr
[junit4] 2> 3296944 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/queue
[junit4] 2> 3296944 DEBUG (OverseerAutoScalingTriggerThread-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:55105_solr, 127.0.0.1:41853_solr, 127.0.0.1:43657_solr]
[junit4] 2> 3296945 INFO (jetty-launcher-52587-thread-4) [n:127.0.0.1:48734_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:48734_solr
[junit4] 2> 3296945 DEBUG (OverseerAutoScalingTriggerThread-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion -1
[junit4] 2> 3296945 DEBUG (OverseerAutoScalingTriggerThread-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
[junit4] 2> 3296945 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 3296955 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 3296955 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 3296956 INFO (zkCallback-52613-thread-1-processing-n:127.0.0.1:55105_solr) [n:127.0.0.1:55105_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (4)
[junit4] 2> 3296956 INFO (zkCallback-52611-thread-1-processing-n:127.0.0.1:41853_solr) [n:127.0.0.1:41853_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (4)
[junit4] 2> 3296969 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
[junit4] 2> 3296969 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 3296969 DEBUG (OverseerStateUpdate-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.Overseer processMessage: queueSize: 3, message = {
[junit4] 2> "operation":"downnode",
[junit4] 2> "node_name":"127.0.0.1:48734_solr"} current state version: 0
[junit4] 2> 3296969 DEBUG (OverseerStateUpdate-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.o.NodeMutator DownNode state invoked for node: 127.0.0.1:48734_solr
[junit4] 2> 3296972 INFO (zkCallback-52621-thread-2-processing-n:127.0.0.1:48734_solr) [n:127.0.0.1:48734_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 3296976 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 3296976 INFO (zkCallback-52617-thread-1-processing-n:127.0.0.1:43657_solr) [n:127.0.0.1:43657_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (4)
[junit4] 2> 3296982 DEBUG (OverseerAutoScalingTriggerThread-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 3296982 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/autoscaling/events/.auto_add_replicas
[junit4] 2> 3296982 DEBUG (OverseerAutoScalingTriggerThread-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
[junit4] 2> 3296992 DEBUG (ScheduledTrigger-11592-thread-1) [n:127.0.0.1:48734_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
[junit4] 2> 3297037 INFO (jetty-launcher-52587-thread-2) [n:127.0.0.1:41853_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41853.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76dd2fbc
[junit4] 2> 3297060 INFO (jetty-launcher-52587-thread-2) [n:127.0.0.1:41853_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41853.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76dd2fbc
[junit4] 2> 3297061 INFO (jetty-launcher-52587-thread-2) [n:127.0.0.1:41853_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41853.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76dd2fbc
[junit4] 2> 3297062 INFO (jetty-launcher-52587-thread-2) [n:127.0.0.1:41853_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.MoveReplicaHDFSTest_BBB8DF46830C4269-001/tempDir-001/node2/.
[junit4] 2> 3297068 INFO (jetty-launcher-52587-thread-1) [n:127.0.0.1:43657_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43657.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76dd2fbc
[junit4] 2> 3297070 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/queue
[junit4] 2> 3297080 INFO (jetty-launcher-52587-thread-1) [n:127.0.0.1:43657_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43657.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76dd2fbc
[junit4] 2> 3297080 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
[junit4] 2> 3297081 INFO (jetty-launcher-52587-thread-1) [n:127.0.0.1:43657_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43657.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76dd2fbc
[junit4] 2> 3297082 INFO (jetty-launcher-52587-thread-1) [n:127.0.0.1:43657_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.MoveReplicaHDFSTest_BBB8DF46830C4269-001/tempDir-001/node1/.
[junit4] 2> 3297095 INFO (jetty-launcher-52587-thread-4) [n:127.0.0.1:48734_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_48734.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76dd2fbc
[junit4] 2> 3297109 INFO (jetty-launcher-52587-thread-4) [n:127.0.0.1:48734_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_48734.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76dd2fbc
[junit4] 2> 3297109 INFO (jetty-launcher-52587-thread-4) [n:127.0.0.1:48734_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_48734.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76dd2fbc
[junit4] 2> 3297110 INFO (jetty-launcher-52587-thread-4) [n:127.0.0.1:48734_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.MoveReplicaHDFSTest_BBB8DF46830C4269-001/tempDir-001/node4/.
[junit4] 2> 3297125 INFO (jetty-launcher-52587-thread-3) [n:127.0.0.1:55105_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_55105.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76dd2fbc
[junit4] 2> 3297138 INFO (jetty-launcher-52587-thread-3) [n:127.0.0.1:55105_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_55105.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76dd2fbc
[junit4] 2> 3297139 INFO (jetty-launcher-52587-thread-3) [n:127.0.0.1:55105_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_55105.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76dd2fbc
[junit4] 2> 3297140 INFO (jetty-launcher-52587-thread-3) [n:127.0.0.1:55105_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.MoveReplicaHDFSTest_BBB8DF46830C4269-001/tempDir-001/node3/.
[junit4] 2> 3297291 INFO (zkConnectionManagerCallback-52632-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3297294 INFO (zkConnectionManagerCallback-52636-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3297295 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/clusterprops.json
[junit4] 2> 3297295 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 3297295 INFO (SUITE-MoveReplicaHDFSTest-seed#[BBB8DF46830C4269]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 3297296 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterstate.json
[junit4] 2> 3297296 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 3297296 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/aliases.json
[junit4] 2> 3297297 INFO (SUITE-MoveReplicaHDFSTest-seed#[BBB8DF46830C4269]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40718/solr ready
[junit4] 2> 3297304 INFO (qtp406252100-112318) [n:127.0.0.1:41853_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :overseerstatus with params action=OVERSEERSTATUS&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3297304 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/collection-queue-work
[junit4] 2> 3297305 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/overseer/collection-queue-work/qnr-0000000000
[junit4] 2> 3297305 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/collection-queue-work
[junit4] 2> 3297305 DEBUG (OverseerCollectionConfigSetProcessor-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.ZkDistributedQueue Found child node with improper name: qnr-0000000000
[junit4] 2> 3297305 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/collection-queue-work
[junit4] 2> 3297305 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/collection-queue-work
[junit4] 2> 3297306 DEBUG (OverseerCollectionConfigSetProcessor-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.ZkDistributedQueue Found child node with improper name: qnr-0000000000
[junit4] 2> 3297306 DEBUG (OverseerCollectionConfigSetProcessor-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.OverseerTaskQueue Returning topN elements: [/overseer/collection-queue-work/qn-0000000000, ]
[junit4] 2> 3297306 DEBUG (OverseerCollectionConfigSetProcessor-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.OverseerTaskProcessor Got 1 tasks from work-queue : [[org.apache.solr.cloud.OverseerTaskQueue$QueueEvent@2e684ecf]]
[junit4] 2> 3297306 DEBUG (OverseerCollectionConfigSetProcessor-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.OverseerTaskProcessor Marked task [/overseer/collection-queue-work/qn-0000000000] as running
[junit4] 2> 3297306 DEBUG (OverseerCollectionConfigSetProcessor-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Get the message id:/overseer/collection-queue-work/qn-0000000000 message:{"operation":"overseerstatus"}
[junit4] 2> 3297306 DEBUG (OverseerThreadFactory-11594-thread-1-processing-n:127.0.0.1:48734_solr) [n:127.0.0.1:48734_solr ] o.a.s.c.OverseerTaskProcessor Runner processing /overseer/collection-queue-work/qn-0000000000
[junit4] 2> 3297306 DEBUG (OverseerThreadFactory-11594-thread-1-processing-n:127.0.0.1:48734_solr) [n:127.0.0.1:48734_solr ] o.a.s.c.a.c.OverseerCollectionMessageHandler OverseerCollectionMessageHandler.processMessage : overseerstatus , {"operation":"overseerstatus"}
[junit4] 2> 3297307 DEBUG (OverseerCollectionConfigSetProcessor-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 1
[junit4] 2> 3297307 DEBUG (OverseerCollectionConfigSetProcessor-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.OverseerTaskProcessor RunningTasks: [/overseer/collection-queue-work/qn-0000000000]
[junit4] 2> 3297307 DEBUG (OverseerCollectionConfigSetProcessor-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
[junit4] 2> 3297307 DEBUG (OverseerCollectionConfigSetProcessor-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
[junit4] 2> 3297307 DEBUG (OverseerCollectionConfigSetProcessor-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: [/overseer/collection-queue-work/qn-0000000000]
[junit4] 2> 3297307 DEBUG (OverseerCollectionConfigSetProcessor-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.OverseerTaskQueue Peeking for top 99 elements. ExcludeSet: [/overseer/collection-queue-work/qn-0000000000],[]
[junit4] 2> 3297314 DEBUG (OverseerThreadFactory-11594-thread-1-processing-n:127.0.0.1:48734_solr) [n:127.0.0.1:48734_solr ] o.a.s.c.OverseerTaskProcessor Completed task:[/overseer/collection-queue-work/qn-0000000000]
[junit4] 2> 3297314 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: data: /solr/overseer/collection-queue-work/qnr-0000000000
[junit4] 2> 3297315 DEBUG (zkCallback-52611-thread-1-processing-n:127.0.0.1:41853_solr) [n:127.0.0.1:41853_solr ] o.a.s.c.OverseerTaskQueue NodeDataChanged fired on path /overseer/collection-queue-work/qnr-0000000000 state SyncConnected latchEventType null
[junit4] 2> 3297315 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/collection-queue-work
[junit4] 2> 3297315 DEBUG (OverseerThreadFactory-11594-thread-1-processing-n:127.0.0.1:48734_solr) [n:127.0.0.1:48734_solr ] o.a.s.c.OverseerTaskProcessor Marked task [/overseer/collection-queue-work/qn-0000000000] as completed.
[junit4] 2> 3297316 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/collection-queue-work
[junit4] 2> 3297320 DEBUG (OverseerThreadFactory-11594-thread-1-processing-n:127.0.0.1:48734_solr) [n:127.0.0.1:48734_solr ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
[junit4] 2> 3297316 INFO (qtp406252100-112318) [n:127.0.0.1:41853_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=OVERSEERSTATUS&wt=javabin&version=2} status=0 QTime=12
[junit4] 2> 3297320 DEBUG (OverseerThreadFactory-11594-thread-1-processing-n:127.0.0.1:48734_solr) [n:127.0.0.1:48734_solr ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
[junit4] 2> 3297320 DEBUG (OverseerThreadFactory-11594-thread-1-processing-n:127.0.0.1:48734_solr) [n:127.0.0.1:48734_solr ] o.a.s.c.OverseerTaskProcessor CompletedTasks: [/overseer/collection-queue-work/qn-0000000000]
[junit4] 2> 3297320 DEBUG (OverseerThreadFactory-11594-thread-1-processing-n:127.0.0.1:48734_solr) [n:127.0.0.1:48734_solr ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: [/overseer/collection-queue-work/qn-0000000000]
[junit4] 2> 3297321 DEBUG (OverseerThreadFactory-11594-thread-1-processing-n:127.0.0.1:48734_solr) [n:127.0.0.1:48734_solr ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Message id:/overseer/collection-queue-work/qn-0000000000 complete, response:{leader=127.0.0.1:48734_solr,overseer_queue_size=0,overseer_work_queue_size=0,overseer_collection_queue_size=2,overseer_operations={am_i_leader={requests=3,errors=0,avgRequestsPerSecond=8.014556379250399,5minRateRequestsPerSecond=0.0,15minRateRequestsPerSecond=0.0,avgTimePerRequest=0.4096186663992751,medianRequestTime=0.440898,75thPcRequestTime=0.554588,95thPcRequestTime=0.554588,99thPcRequestTime=0.554588,999thPcRequestTime=0.554588},downnode={requests=3,errors=0,avgRequestsPerSecond=8.23837989491271,5minRateRequestsPerSecond=0.0,15minRateRequestsPerSecond=0.0,avgTimePerRequest=0.055381,medianRequestTime=0.05671,75thPcRequestTime=0.05821,95thPcRequestTime=0.05821,99thPcRequestTime=0.05821,999thPcRequestTime=0.05821}},collection_operations={am_i_leader={requests=3,errors=0,avgRequestsPerSecond=8.020773782708485,5minRateRequestsPerSecond=0.0,15minRateRequestsPerSecond=0.0,avgTimePerRequest=0.3583706853337792,medianRequestTime=0.249462,75thPcRequestTime=0.683644,95thPcRequestTime=0.683644,99thPcRequestTime=0.683644,999thPcRequestTime=0.683644},overseerstatus={requests=0,errors=0,avgRequestsPerSecond=0.0,5minRateRequestsPerSecond=0.0,15minRateRequestsPerSecond=0.0,avgTimePerRequest=0.0,medianRequestTime=0.0,75thPcRequestTime=0.0,95thPcRequestTime=0.0,99thPcRequestTime=0.0,999thPcRequestTime=0.0}},overseer_queue={},overseer_internal_queue={peek={avgRequestsPerSecond=2.6949622031281515,5minRateRequestsPerSecond=0.0,15minRateRequestsPerSecond=0.0,avgTimePerRequest=0.944358,medianRequestTime=0.944358,75thPcRequestTime=0.944358,95thPcRequestTime=0.944358,99thPcRequestTime=0.944358,999thPcRequestTime=0.944358}},collection_queue={peektopn_wait2000={avgRequestsPerSecond=2.687769917950125,5minRateRequestsPerSecond=0.0,15minRateRequestsPerSecond=0.0,avgTimePerRequest=369.990197,medianRequestTime=369.990197,75thPcRequestTime=369.990197,95thPcRequestTime=369.990197,99thPcRequestTime=369.990197,999thPcRequestTime=369.990197}}}
[junit4] 1> Formatting using clusterid: testClusterID
[junit4] 2> 3297366 WARN (SUITE-MoveReplicaHDFSTest-seed#[BBB8DF46830C4269]-worker) [ ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
[junit4] 2> 3297377 WARN (SUITE-MoveReplicaHDFSTest-seed#[BBB8DF46830C4269]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 3297379 INFO (SUITE-MoveReplicaHDFSTest-seed#[BBB8DF46830C4269]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 3297393 INFO (SUITE-MoveReplicaHDFSTest-seed#[BBB8DF46830C4269]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/hdfs to ./temp/Jetty_localhost_57691_hdfs____uabnh2/webapp
[junit4] 2> 3297993 DEBUG (ScheduledTrigger-11592-thread-1) [n:127.0.0.1:48734_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
[junit4] 2> 3298994 DEBUG (ScheduledTrigger-11592-thread-2) [n:127.0.0.1:48734_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
[junit4] 2> 3298999 INFO (SUITE-MoveReplicaHDFSTest-seed#[BBB8DF46830C4269]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:57691
[junit4] 2> 3299093 WARN (SUITE-MoveReplicaHDFSTest-seed#[BBB8DF46830C4269]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 3299095 INFO (SUITE-MoveReplicaHDFSTest-seed#[BBB8DF46830C4269]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 3299111 INFO (SUITE-MoveReplicaHDFSTest-seed#[BBB8DF46830C4269]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_42632_datanode____.v04pfm/webapp
[junit4] 2> 3299312 DEBUG (OverseerCollectionConfigSetProcessor-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
[junit4] 2> 3299355 DEBUG (OverseerCollectionConfigSetProcessor-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
[junit4] 2> 3299355 INFO (OverseerCollectionConfigSetProcessor-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 3299356 DEBUG (OverseerCollectionConfigSetProcessor-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
[junit4] 2> 3299356 DEBUG (OverseerCollectionConfigSetProcessor-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
[junit4] 2> 3299356 DEBUG (OverseerCollectionConfigSetProcessor-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
[junit4] 2> 3299356 DEBUG (OverseerCollectionConfigSetProcessor-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
[junit4] 2> 3299356 DEBUG (OverseerCollectionConfigSetProcessor-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
[junit4] 2> 3299572 INFO (SUITE-MoveReplicaHDFSTest-seed#[BBB8DF46830C4269]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:42632
[junit4] 2> 3299616 WARN (SUITE-MoveReplicaHDFSTest-seed#[BBB8DF46830C4269]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 3299617 INFO (SUITE-MoveReplicaHDFSTest-seed#[BBB8DF46830C4269]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 3299633 INFO (SUITE-MoveReplicaHDFSTest-seed#[BBB8DF46830C4269]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_48441_datanode____70wx78/webapp
[junit4] 2> 3299671 ERROR (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.MoveReplicaHDFSTest_BBB8DF46830C4269-001/tempDir-002/hdfsBaseDir/data/data1/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.MoveReplicaHDFSTest_BBB8DF46830C4269-001/tempDir-002/hdfsBaseDir/data/data2/]] heartbeating to localhost/127.0.0.1:59007) [ ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
[junit4] 2> 3299682 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x48a12887cee750: from storage DS-43b92334-213c-436f-812a-ab736cb21932 node DatanodeRegistration(127.0.0.1:45428, datanodeUuid=c7925c8d-dd94-439a-a4e5-25c680a3d46d, infoPort=36108, infoSecurePort=0, ipcPort=39562, storageInfo=lv=-56;cid=testClusterID;nsid=46744604;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
[junit4] 2> 3299682 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x48a12887cee750: from storage DS-c8da9a98-f76d-44aa-b3c7-cdd2a4be6b65 node DatanodeRegistration(127.0.0.1:45428, datanodeUuid=c7925c8d-dd94-439a-a4e5-25c680a3d46d, infoPort=36108, infoSecurePort=0, ipcPort=39562, storageInfo=lv=-56;cid=testClusterID;nsid=46744604;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 3299995 DEBUG (ScheduledTrigger-11592-thread-1) [n:127.0.0.1:48734_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
[junit4] 2> 3300091 INFO (SUITE-MoveReplicaHDFSTest-seed#[BBB8DF46830C4269]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:48441
[junit4] 2> 3300270 ERROR (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.MoveReplicaHDFSTest_BBB8DF46830C4269-001/tempDir-002/hdfsBaseDir/data/data3/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.MoveReplicaHDFSTest_BBB8DF46830C4269-001/tempDir-002/hdfsBaseDir/data/data4/]] heartbeating to localhost/127.0.0.1:59007) [ ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
[junit4] 2> 3300305 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x48a128acf315a0: from storage DS-97b61b1e-058a-47c8-8774-a67c6cc2cba8 node DatanodeRegistration(127.0.0.1:58534, datanodeUuid=2e83ecb6-e381-4fcd-9b92-4127b111a153, infoPort=38072, infoSecurePort=0, ipcPort=44057, storageInfo=lv=-56;cid=testClusterID;nsid=46744604;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
[junit4] 2> 3300305 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x48a128acf315a0: from storage DS-f3c999ff-cba0-498c-bfa1-74b0f96c5932 node DatanodeRegistration(127.0.0.1:58534, datanodeUuid=2e83ecb6-e381-4fcd-9b92-4127b111a153, infoPort=38072, infoSecurePort=0, ipcPort=44057, storageInfo=lv=-56;cid=testClusterID;nsid=46744604;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 3300457 INFO (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[BBB8DF46830C4269]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testFailedMove
[junit4] 2> 3300458 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/clusterprops.json
[junit4] 2> 3300459 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 3300459 INFO (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[BBB8DF46830C4269]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 3300459 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterstate.json
[junit4] 2> 3300459 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 3300460 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/aliases.json
[junit4] 2> 3300461 INFO (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[BBB8DF46830C4269]) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T22:27:37+01:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 3300469 INFO (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[BBB8DF46830C4269]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 3300469 INFO (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[BBB8DF46830C4269]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 3300469 INFO (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[BBB8DF46830C4269]) [ ] o.e.j.s.session Scavenging every 600000ms
[junit4] 2> 3300469 INFO (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[BBB8DF46830C4269]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@56ef408{/solr,null,AVAILABLE}
[junit4] 2> 3300470 INFO (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[BBB8DF46830C4269]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@4f2c19dc{HTTP/1.1,[http/1.1]}{127.0.0.1:44059}
[junit4] 2> 3300470 INFO (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[BBB8DF46830C4269]) [ ] o.e.j.s.Server Started @3303578ms
[junit4] 2> 3300470 INFO (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[BBB8DF46830C4269]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=44059}
[junit4] 2> 3300470 ERROR (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[BBB8DF46830C4269]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3300470 INFO (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[BBB8DF46830C4269]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 8.0.0
[junit4] 2> 3300471 INFO (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[BBB8DF46830C4269]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3300471 INFO (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[BBB8DF46830C4269]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3300471 INFO (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[BBB8DF46830C4269]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-03-04T19:24:50.427Z
[junit4] 2> 3300475 INFO (zkConnectionManagerCallback-52639-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3300476 INFO (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[BBB8DF46830C4269]) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 3300494 INFO (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[BBB8DF46830C4269]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40718/solr
[junit4] 2> 3300496 INFO (zkConnectionManagerCallback-52643-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3300498 DEBUG (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[BBB8DF46830C4269]) [n:127.0.0.1:44059_solr ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$263/752094002@3a959c2b
[junit4] 2> 3300499 INFO (zkConnectionManagerCallback-52645-thread-1-processing-n:127.0.0.1:44059_solr) [n:127.0.0.1:44059_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3300506 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/clusterprops.json
[junit4] 2> 3300507 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 3300507 INFO (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[BBB8DF46830C4269]) [n:127.0.0.1:44059_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 3300507 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterstate.json
[junit4] 2> 3300507 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 3300508 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/aliases.json
[junit4] 2> 3300508 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/security.json
[junit4] 2> 3300510 INFO (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[BBB8DF46830C4269]) [n:127.0.0.1:44059_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 3300510 DEBUG (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[BBB8DF46830C4269]) [n:127.0.0.1:44059_solr ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/73397372091432977-127.0.0.1:44059_solr-n_0000000004
[junit4] 2> 3300510 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/overseer_elect/election/73397372091432971-127.0.0.1:41853_solr-n_0000000003
[junit4] 2> 3300511 DEBUG (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[BBB8DF46830C4269]) [n:127.0.0.1:44059_solr ] o.a.s.c.LeaderElector Watching path /overseer_elect/election/73397372091432971-127.0.0.1:41853_solr-n_0000000003 to know if I could be the leader
[junit4] 2> 3300511 DEBUG (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[BBB8DF46830C4269]) [n:127.0.0.1:44059_solr ] o.a.s.c.ZkController Publish node=127.0.0.1:44059_solr as DOWN
[junit4] 2> 3300512 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/queue
[junit4] 2> 3300512 INFO (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[BBB8DF46830C4269]) [n:127.0.0.1:44059_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:44059_solr
[junit4] 2> 3300512 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
[junit4] 2> 3300513 DEBUG (OverseerStateUpdate-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"downnode",
[junit4] 2> "node_name":"127.0.0.1:44059_solr"} current state version: 0
[junit4] 2> 3300513 DEBUG (OverseerStateUpdate-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.o.NodeMutator DownNode state invoked for node: 127.0.0.1:44059_solr
[junit4] 2> 3300513 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 3300513 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 3300513 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 3300514 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 3300514 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 3300514 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 3300514 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 3300514 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 3300514 INFO (zkCallback-52613-thread-1-processing-n:127.0.0.1:55105_solr) [n:127.0.0.1:55105_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 3300514 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 3300514 INFO (zkCallback-52621-thread-2-processing-n:127.0.0.1:48734_solr) [n:127.0.0.1:48734_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 3300515 INFO (zkCallback-52611-thread-1-processing-n:127.0.0.1:41853_solr) [n:127.0.0.1:41853_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 3300515 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 3300515 INFO (zkCallback-52635-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 3300515 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 3300515 INFO (zkCallback-52635-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 3300515 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 3300516 INFO (zkCallback-52617-thread-1-processing-n:127.0.0.1:43657_solr) [n:127.0.0.1:43657_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 3300516 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 3300517 INFO (zkCallback-52644-thread-1-processing-n:127.0.0.1:44059_solr) [n:127.0.0.1:44059_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 3300615 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/queue
[junit4] 2> 3300616 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
[junit4] 2> 3300621 INFO (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[BBB8DF46830C4269]) [n:127.0.0.1:44059_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_44059.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76dd2fbc
[junit4] 2> 3300635 INFO (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[BBB8DF46830C4269]) [n:127.0.0.1:44059_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_44059.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76dd2fbc
[junit4] 2> 3300636 INFO (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[BBB8DF46830C4269]) [n:127.0.0.1:44059_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_44059.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76dd2fbc
[junit4] 2> 3300638 INFO (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[BBB8DF46830C4269]) [n:127.0.0.1:44059_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.MoveReplicaHDFSTest_BBB8DF46830C4269-001/tempDir-001/node5/.
[junit4] 2> 3300719 INFO (zkConnectionManagerCallback-52649-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3300726 INFO (qtp406252100-112327) [n:127.0.0.1:41853_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=2&collection.configName=conf1&autoAddReplicas=false&name=MoveReplicaHDFSTest_failed_coll_true&nrtReplicas=2&action=CREATE&numShards=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3300727 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/collection-queue-work
[junit4] 2> 3300727 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/overseer/collection-queue-work/qnr-0000000002
[junit4] 2> 3300727 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/collection-queue-work
[junit4] 2> 3300727 DEBUG (OverseerCollectionConfigSetProcessor-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.ZkDistributedQueue Found child node with improper name: qnr-0000000002
[junit4] 2> 3300727 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/collection-queue-work
[junit4] 2> 3300728 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/collection-queue-work
[junit4] 2> 3300728 DEBUG (OverseerCollectionConfigSetProcessor-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.ZkDistributedQueue Found child node with improper name: qnr-0000000002
[junit4] 2> 3300728 DEBUG (OverseerCollectionConfigSetProcessor-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.OverseerTaskQueue Returning topN elements: [/overseer/collection-queue-work/qn-0000000002, ]
[junit4] 2> 3300728 DEBUG (OverseerCollectionConfigSetProcessor-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.OverseerTaskProcessor Got 1 tasks from work-queue : [[org.apache.solr.cloud.OverseerTaskQueue$QueueEvent@2e684ed1]]
[junit4] 2> 3300728 DEBUG (OverseerCollectionConfigSetProcessor-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.OverseerTaskProcessor Marked task [/overseer/collection-queue-work/qn-0000000002] as running
[junit4] 2> 3300729 DEBUG (OverseerCollectionConfigSetProcessor-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Get the message id:/overseer/collection-queue-work/qn-0000000002 message:{
[junit4] 2> "name":"MoveReplicaHDFSTest_failed_coll_true",
[junit4] 2> "fromApi":"true",
[junit4] 2> "replicationFactor":"2",
[junit4] 2> "collection.configName":"conf1",
[junit4] 2> "numShards":"2",
[junit4] 2> "autoAddReplicas":"false",
[junit4] 2> "nrtReplicas":"2",
[junit4] 2> "stateFormat":"2",
[junit4] 2> "operation":"create"}
[junit4] 2> 3300729 DEBUG (OverseerThreadFactory-11594-thread-2-processing-n:127.0.0.1:48734_solr) [n:127.0.0.1:48734_solr ] o.a.s.c.OverseerTaskProcessor Runner processing /overseer/collection-queue-work/qn-0000000002
[junit4] 2> 3300729 DEBUG (OverseerThreadFactory-11594-thread-2-processing-n:127.0.0.1:48734_solr) [n:127.0.0.1:48734_solr ] o.a.s.c.a.c.OverseerCollectionMessageHandler OverseerCollectionMessageHandler.processMessage : create , {
[junit4] 2> "name":"MoveReplicaHDFSTest_failed_coll_true",
[junit4] 2> "fromApi":"true",
[junit4] 2> "replicationFactor":"2",
[junit4] 2> "collection.configName":"conf1",
[junit4] 2> "numShards":"2",
[junit4] 2> "autoAddReplicas":"false",
[junit4] 2> "nrtReplicas":"2",
[junit4] 2> "stateFormat":"2",
[junit4] 2> "operation":"create"}
[junit4] 2> 3300729 INFO (OverseerThreadFactory-11594-thread-2-processing-n:127.0.0.1:48734_solr) [n:127.0.0.1:48734_solr ] o.a.s.c.a.c.CreateCollectionCmd Create collection MoveReplicaHDFSTest_failed_coll_true
[junit4] 2> 3300729 DEBUG (OverseerCollectionConfigSetProcessor-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 1
[junit4] 2> 3300729 DEBUG (OverseerCollectionConfigSetProcessor-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.OverseerTaskProcessor RunningTasks: [/overseer/collection-queue-work/qn-0000000002]
[junit4] 2> 3300729 DEBUG (OverseerCollectionConfigSetProcessor-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
[junit4] 2> 3300729 DEBUG (OverseerCollectionConfigSetProcessor-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
[junit4] 2> 3300729 DEBUG (OverseerCollectionConfigSetProcessor-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: [/overseer/collection-queue-work/qn-0000000002]
[junit4] 2> 3300729 DEBUG (OverseerCollectionConfigSetProcessor-73397372091432972-127.0.0.1:48734_solr-n_0000000000) [n:127.0.0.1:48734_solr ] o.a.s.c.OverseerTaskQueu
[...truncated too long message...]
rqgf%2Fzx-n_0000000001) [n:127.0.0.1:45749_rqgf%2Fzx ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 4552486 INFO (OverseerStateUpdate-73397453256327176-127.0.0.1:45749_rqgf%2Fzx-n_0000000001) [n:127.0.0.1:45749_rqgf%2Fzx ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:45749_rqgf%2Fzx
[junit4] 2> 4552488 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[BBB8DF46830C4269]) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@53034d68{SSL,[ssl, http/1.1]}{127.0.0.1:0}
[junit4] 2> 4552489 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[BBB8DF46830C4269]) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@249353ed{/rqgf/zx,null,UNAVAILABLE}
[junit4] 2> 4552489 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[BBB8DF46830C4269]) [ ] o.e.j.s.session Stopped scavenging
[junit4] 2> 4552489 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[BBB8DF46830C4269]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testPartialReplicationAfterPeerSync
[junit4] 2> 4552489 ERROR (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[BBB8DF46830C4269]) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 4552492 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[BBB8DF46830C4269]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:48984 48984
[junit4] 2> 4552662 INFO (Thread-12310) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:48984 48984
[junit4] 2> 4552669 WARN (Thread-12310) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 30 /solr/aliases.json
[junit4] 2> 20 /solr/collections/source_collection/terms/shard1
[junit4] 2> 12 /solr/collections/tmp_collection/terms/shard1
[junit4] 2> 6 /solr/configs/conf1
[junit4] 2> 5 /solr/collections/source_collection/leaders/shard1/leader
[junit4] 2> 4 /solr/security.json
[junit4] 2> 2 /solr/collections/tmp_collection/leaders/shard1/leader
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 30 /solr/clusterprops.json
[junit4] 2> 30 /solr/clusterstate.json
[junit4] 2> 5 /solr/collections/source_collection/cdcr/state/buffer
[junit4] 2> 5 /solr/collections/source_collection/cdcr/state/process
[junit4] 2> 4 /solr/collections/source_collection/state.json
[junit4] 2> 3 /solr/collections/source_collection/leader_elect/shard1/election/73397453256327176-core_node3-n_0000000000
[junit4] 2> 2 /solr/overseer_elect/election/73397453256327176-127.0.0.1:45749_rqgf%2Fzx-n_0000000001
[junit4] 2> 2 /solr/collections/tmp_collection/state.json
[junit4] 2> 2 /solr/collections/tmp_collection/cdcr/state/buffer
[junit4] 2> 2 /solr/autoscaling.json
[junit4] 2> 2 /solr/collections/tmp_collection/cdcr/state/process
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 30 /solr/live_nodes
[junit4] 2> 30 /solr/collections
[junit4] 2> 2 /solr/overseer/queue
[junit4] 2> 2 /solr/autoscaling/events/.auto_add_replicas
[junit4] 2> 2 /solr/overseer/queue-work
[junit4] 2> 2 /solr/overseer/collection-queue-work
[junit4] 2>
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.cdcr.CdcrReplicationHandlerTest_BBB8DF46830C4269-001
[junit4] 2> Mar 04, 2018 7:45:42 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=SimpleText, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@4d306694), locale=en-NZ, timezone=Etc/GMT+5
[junit4] 2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=108257152,total=526385152
[junit4] 2> NOTE: All tests run in this JVM: [TestSchemaVersionResource, SmileWriterTest, TestMaxScoreQueryParser, HdfsDirectoryFactoryTest, TestMinMaxOnMultiValuedField, DirectUpdateHandlerOptimizeTest, ShardSplitTest, TestCloudPseudoReturnFields, UpdateLogTest, LeaderInitiatedRecoveryOnShardRestartTest, TestPayloadScoreQParserPlugin, SortSpecParsingTest, BadCopyFieldTest, SignatureUpdateProcessorFactoryTest, CdcrBootstrapTest, TestCloudInspectUtil, SoftAutoCommitTest, TestPostingsSolrHighlighter, AsyncCallRequestStatusResponseTest, FieldMutatingUpdateProcessorTest, ShardRoutingCustomTest, SchemaApiFailureTest, OutOfBoxZkACLAndCredentialsProvidersTest, MetricUtilsTest, HdfsRecoveryZkTest, ZkShardTermsTest, DeleteNodeTest, TestSystemCollAutoCreate, TestExportWriter, HdfsThreadLeakTest, HttpSolrCallGetCoreTest, TestConfigSetsAPIZkFailure, PreAnalyzedUpdateProcessorTest, SearchHandlerTest, SuggesterTest, LargeFieldTest, TestPerFieldSimilarityWithDefaultOverride, DeleteStatusTest, TestReloadAndDeleteDocs, EnumFieldTest, TestSolrConfigHandler, TestLegacyFieldCache, HighlighterMaxOffsetTest, PropertiesRequestHandlerTest, SliceStateTest, ScriptEngineTest, TestMacroExpander, SolrGangliaReporterTest, ConfigSetsAPITest, ReplicationFactorTest, BasicZkTest, TestFieldCacheSortRandom, TestSimpleTrackingShardHandler, QueryElevationComponentTest, TestOnReconnectListenerSupport, SchemaVersionSpecificBehaviorTest, TestCollationFieldDocValues, TestSolrCloudWithSecureImpersonation, HighlighterTest, TestConfigSets, TestDistribIDF, TestSQLHandlerNonCloud, TestFastLRUCache, TestNamedUpdateProcessors, TestRTGBase, TestFileDictionaryLookup, TestDistributedStatsComponentCardinality, TestReversedWildcardFilterFactory, TestStressLiveNodes, TestPerFieldSimilarity, TestReloadDeadlock, TestNumericRangeQuery64, TestHashQParserPlugin, TestSolr4Spatial, TestReplicationHandler, CoreMergeIndexesAdminHandlerTest, TestPivotHelperCode, TestNestedDocsSort, TestSubQueryTransformerDistrib, BadComponentTest, LoggingHandlerTest, BitVectorTest, TestSolrCLIRunExample, TestEmbeddedSolrServerAdminHandler, TestRebalanceLeaders, TestSchemaManager, OverseerTest, BJQParserTest, TestFoldingMultitermQuery, StatsReloadRaceTest, TestChildDocTransformer, TestReplicaProperties, SSLMigrationTest, DocExpirationUpdateProcessorFactoryTest, ResourceLoaderTest, TestSegmentSorting, RequiredFieldsTest, SOLR749Test, SortByFunctionTest, UpdateRequestProcessorFactoryTest, TestCharFilters, MetricTriggerTest, BasicFunctionalityTest, DocumentBuilderTest, SystemInfoHandlerTest, TestFunctionQuery, TestGenericDistributedQueue, IndexSchemaTest, TestXIncludeConfig, PeerSyncTest, TestDirectoryFactory, ConcurrentDeleteAndCreateCollectionTest, ScheduledTriggerTest, TestAuthenticationFramework, AnalyticsQueryTest, XsltUpdateRequestHandlerTest, TestSolrIndexConfig, TestNoOpRegenerator, DistributedMLTComponentTest, TestBulkSchemaConcurrent, QueryParsingTest, TestLegacyTerms, DistributedIntervalFacetingTest, TestClusterProperties, TestSolrJ, JsonLoaderTest, TestBM25SimilarityFactory, TestReqParamsAPI, TestCoreContainer, SimpleFacetsTest, SolrCoreTest, SolrCmdDistributorTest, ConvertedLegacyTest, TestFiltering, SimplePostToolTest, TestExtendedDismaxParser, SpellCheckCollatorTest, SuggesterFSTTest, DocValuesTest, TestStressLucene, TestTrie, SpatialFilterTest, TestCSVLoader, PolyFieldTest, WordBreakSolrSpellCheckerTest, TestPseudoReturnFields, TestUpdate, QueryEqualityTest, DefaultValueUpdateProcessorTest, TestRemoteStreaming, DistanceFunctionTest, SolrInfoBeanTest, CacheHeaderTest, TestSurroundQueryParser, TestQueryUtils, DocumentAnalysisRequestHandlerTest, PrimitiveFieldTypeTest, FileBasedSpellCheckerTest, TermsComponentTest, XmlUpdateRequestHandlerTest, TestValueSourceCache, PathHierarchyTokenizerFactoryTest, TermVectorComponentTest, FastVectorHighlighterTest, TestJmxIntegration, ReturnFieldsTest, MBeansHandlerTest, UniqFieldsUpdateProcessorFactoryTest, BinaryUpdateRequestHandlerTest, TestPartialUpdateDeduplication, PingRequestHandlerTest, CSVRequestHandlerTest, TestComponentsName, HighlighterConfigTest, TestQuerySenderListener, UpdateParamsTest, AlternateDirectoryTest, TestQuerySenderNoQuery, CopyFieldTest, ResponseLogComponentTest, SolrIndexConfigTest, TestStressRecovery, PreAnalyzedFieldTest, TestSystemIdResolver, PrimUtilsTest, SpellingQueryConverterTest, DOMUtilTest, ClusterStateTest, TestLRUCache, TestUtils, TestDocumentBuilder, FileUtilsTest, CircularListTest, SolrTestCaseJ4Test, TestCursorMarkWithoutUniqueKey, TestDistributedMissingSort, TestHighlightDedupGrouping, TestLuceneIndexBackCompat, TestEmbeddedSolrServerSchemaAPI, TestJettySolrRunner, ConnectionReuseTest, AddReplicaTest, AssignBackwardCompatibilityTest, ChaosMonkeyNothingIsSafeWithPullReplicasTest, HealthCheckHandlerTest, HttpPartitionOnCommitTest, LegacyCloudClusterPropTest, MissingSegmentRecoveryTest, MoveReplicaHDFSTest, TestExclusionRuleCollectionAccess, TestLeaderInitiatedRecoveryThread, TestPullReplicaErrorHandling, TestRandomFlRTGCloud, TestTolerantUpdateProcessorCloud, TestTolerantUpdateProcessorRandomCloud, TestUtilizeNode, TriLevelCompositeIdRoutingTest, AssignTest, CollectionReloadTest, CollectionTooManyReplicasTest, CollectionsAPIDistributedZkTest, TestCollectionAPI, AutoAddReplicasIntegrationTest, AutoAddReplicasPlanActionTest, AutoScalingHandlerTest, ExecutePlanActionTest, SearchRateTriggerTest, TestComputePlanAction, TestExecutePlanAction, TestLargeCluster, CdcrReplicationHandlerTest]
[junit4] Completed [599/780 (2!)] on J1 in 96.09s, 5 tests, 1 failure <<< FAILURES!
[...truncated 48466 lines...]
[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1493 - Still
Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1493/
6 tests failed.
FAILED: org.apache.solr.cloud.api.collections.ShardSplitTest.testSplitAfterFailedSplit
Error Message:
expected:<1> but was:<2>
Stack Trace:
java.lang.AssertionError: expected:<1> but was:<2>
at __randomizedtesting.SeedInfo.seed([CA8C715A4ADC7334:33C1E2F576A93EBE]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:472)
at org.junit.Assert.assertEquals(Assert.java:456)
at org.apache.solr.cloud.api.collections.ShardSplitTest.testSplitAfterFailedSplit(ShardSplitTest.java:283)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.api.collections.ShardSplitTest.testSplitShardWithRule
Error Message:
Error from server at http://127.0.0.1:37164: Could not find collection : shardSplitWithRule
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:37164: Could not find collection : shardSplitWithRule
at __randomizedtesting.SeedInfo.seed([CA8C715A4ADC7334:4B82A0E8D72E281F]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1104)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:884)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:817)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
at org.apache.solr.cloud.api.collections.ShardSplitTest.testSplitShardWithRule(ShardSplitTest.java:529)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest.test
Error Message:
expected:<0> but was:<1>
Stack Trace:
java.lang.AssertionError: expected:<0> but was:<1>
at __randomizedtesting.SeedInfo.seed([CA8C715A4ADC7334:42D84E80E4201ECC]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:472)
at org.junit.Assert.assertEquals(Assert.java:456)
at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test(ChaosMonkeySafeLeaderTest.java:146)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.hdfs.HdfsRestartWhileUpdatingTest.test
Error Message:
There are still nodes recoverying - waited for 320 seconds
Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 320 seconds
at __randomizedtesting.SeedInfo.seed([CA8C715A4ADC7334:42D84E80E4201ECC]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:185)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:921)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1478)
at org.apache.solr.cloud.RestartWhileUpdatingTest.test(RestartWhileUpdatingTest.java:144)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsRestartWhileUpdatingTest
Error Message:
7 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsRestartWhileUpdatingTest: 1) Thread[id=11260, name=searcherExecutor-1553-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=11159, name=searcherExecutor-1525-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=11091, name=searcherExecutor-1503-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=11122, name=searcherExecutor-1514-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=11289, name=searcherExecutor-1564-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=11055, name=searcherExecutor-1492-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=11231, name=searcherExecutor-1542-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 7 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsRestartWhileUpdatingTest:
1) Thread[id=11260, name=searcherExecutor-1553-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=11159, name=searcherExecutor-1525-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=11091, name=searcherExecutor-1503-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=11122, name=searcherExecutor-1514-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=11289, name=searcherExecutor-1564-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=11055, name=searcherExecutor-1492-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=11231, name=searcherExecutor-1542-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([CA8C715A4ADC7334]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsRestartWhileUpdatingTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=11260, name=searcherExecutor-1553-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=11159, name=searcherExecutor-1525-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=11091, name=searcherExecutor-1503-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=11122, name=searcherExecutor-1514-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=11289, name=searcherExecutor-1564-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=11055, name=searcherExecutor-1492-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=11231, name=searcherExecutor-1542-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=11260, name=searcherExecutor-1553-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=11159, name=searcherExecutor-1525-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=11091, name=searcherExecutor-1503-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=11122, name=searcherExecutor-1514-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=11289, name=searcherExecutor-1564-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=11055, name=searcherExecutor-1492-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=11231, name=searcherExecutor-1542-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([CA8C715A4ADC7334]:0)
Build Log:
[...truncated 12952 lines...]
[junit4] Suite: org.apache.solr.cloud.api.collections.ShardSplitTest
[junit4] 2> 1116257 INFO (SUITE-ShardSplitTest-seed#[CA8C715A4ADC7334]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.api.collections.ShardSplitTest_CA8C715A4ADC7334-001/init-core-data-001
[junit4] 2> 1116258 INFO (SUITE-ShardSplitTest-seed#[CA8C715A4ADC7334]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 1116259 INFO (SUITE-ShardSplitTest-seed#[CA8C715A4ADC7334]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
[junit4] 2> 1116259 INFO (SUITE-ShardSplitTest-seed#[CA8C715A4ADC7334]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
[junit4] IGNOR/A 0.00s J0 | ShardSplitTest.test
[junit4] > Assumption #1: 'badapple' test group is disabled (@BadApple(bugUrl=https://issues.apache.org/jira/browse/SOLR-12028))
[junit4] 2> 1116342 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 1116353 INFO (Thread-974) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1116353 INFO (Thread-974) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 1116374 ERROR (Thread-974) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 1116463 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.c.ZkTestServer start zk server on port:43741
[junit4] 2> 1116487 INFO (zkConnectionManagerCallback-269-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1116560 INFO (zkConnectionManagerCallback-271-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1116576 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 1116595 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4] 2> 1116597 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 1116630 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 1116632 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 1116633 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 1116646 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 1116647 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 1116648 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 1116649 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 1116683 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 1116691 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use TLOG replicas unless explicitly asked otherwise
[junit4] 2> 1117286 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T18:27:37-03:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 1117287 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 1117287 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 1117287 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.e.j.s.session Scavenging every 600000ms
[junit4] 2> 1117287 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1032370f{/,null,AVAILABLE}
[junit4] 2> 1117305 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5b718f58{HTTP/1.1,[http/1.1]}{127.0.0.1:37555}
[junit4] 2> 1117305 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.e.j.s.Server Started @1131084ms
[junit4] 2> 1117305 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.api.collections.ShardSplitTest_CA8C715A4ADC7334-001/tempDir-001/control/data, hostContext=/, hostPort=37555, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.api.collections.ShardSplitTest_CA8C715A4ADC7334-001/control-001/cores}
[junit4] 2> 1117306 ERROR (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1117306 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.0.0
[junit4] 2> 1117306 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1117306 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1117306 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-03-03T23:17:17.390Z
[junit4] 2> 1117318 INFO (zkConnectionManagerCallback-273-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1117320 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1117320 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.api.collections.ShardSplitTest_CA8C715A4ADC7334-001/control-001/solr.xml
[junit4] 2> 1117366 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 1117366 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 1117502 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@f1b3f82, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1117548 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:43741/solr
[junit4] 2> 1117625 INFO (zkConnectionManagerCallback-277-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1117816 INFO (zkConnectionManagerCallback-279-thread-1-processing-n:127.0.0.1:37555_) [n:127.0.0.1:37555_ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1118976 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [n:127.0.0.1:37555_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1118977 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [n:127.0.0.1:37555_ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:37555_
[junit4] 2> 1119189 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [n:127.0.0.1:37555_ ] o.a.s.c.Overseer Overseer (id=72218579117735940-127.0.0.1:37555_-n_0000000000) starting
[junit4] 2> 1119652 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [n:127.0.0.1:37555_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37555_
[junit4] 2> 1119709 INFO (OverseerStateUpdate-72218579117735940-127.0.0.1:37555_-n_0000000000) [n:127.0.0.1:37555_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1120069 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [n:127.0.0.1:37555_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@f1b3f82
[junit4] 2> 1120106 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [n:127.0.0.1:37555_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@f1b3f82
[junit4] 2> 1120106 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [n:127.0.0.1:37555_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@f1b3f82
[junit4] 2> 1120120 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [n:127.0.0.1:37555_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.api.collections.ShardSplitTest_CA8C715A4ADC7334-001/control-001/cores
[junit4] 2> 1120350 INFO (zkConnectionManagerCallback-285-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1120363 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1120364 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:43741/solr ready
[junit4] 2> 1120367 INFO (qtp1333520522-2161) [n:127.0.0.1:37555_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:37555_&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1120469 INFO (OverseerThreadFactory-601-thread-1-processing-n:127.0.0.1:37555_) [n:127.0.0.1:37555_ ] o.a.s.c.a.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 1120745 INFO (qtp1333520522-2165) [n:127.0.0.1:37555_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 1120765 INFO (qtp1333520522-2165) [n:127.0.0.1:37555_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1121004 INFO (zkCallback-278-thread-1-processing-n:127.0.0.1:37555_) [n:127.0.0.1:37555_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1122020 INFO (qtp1333520522-2165) [n:127.0.0.1:37555_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 1122136 INFO (qtp1333520522-2165) [n:127.0.0.1:37555_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 1122562 INFO (qtp1333520522-2165) [n:127.0.0.1:37555_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 1122691 INFO (qtp1333520522-2165) [n:127.0.0.1:37555_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 1122763 INFO (qtp1333520522-2165) [n:127.0.0.1:37555_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@f1b3f82
[junit4] 2> 1122793 INFO (qtp1333520522-2165) [n:127.0.0.1:37555_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1122793 INFO (qtp1333520522-2165) [n:127.0.0.1:37555_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.api.collections.ShardSplitTest_CA8C715A4ADC7334-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.api.collections.ShardSplitTest_CA8C715A4ADC7334-001/control-001/cores/control_collection_shard1_replica_n1/data/]
[junit4] 2> 1122831 INFO (qtp1333520522-2165) [n:127.0.0.1:37555_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=24, maxMergeAtOnceExplicit=14, maxMergedSegmentMB=60.7900390625, floorSegmentMB=1.7802734375, forceMergeDeletesPctAllowed=1.0174268134711073, segmentsPerTier=46.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
[junit4] 2> 1123106 WARN (qtp1333520522-2165) [n:127.0.0.1:37555_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1123522 INFO (qtp1333520522-2165) [n:127.0.0.1:37555_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1123522 INFO (qtp1333520522-2165) [n:127.0.0.1:37555_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1123523 INFO (qtp1333520522-2165) [n:127.0.0.1:37555_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1123523 INFO (qtp1333520522-2165) [n:127.0.0.1:37555_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1123524 INFO (qtp1333520522-2165) [n:127.0.0.1:37555_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@4c68dfd3
[junit4] 2> 1123525 INFO (qtp1333520522-2165) [n:127.0.0.1:37555_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@2856b367[control_collection_shard1_replica_n1] main]
[junit4] 2> 1123542 INFO (qtp1333520522-2165) [n:127.0.0.1:37555_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1123542 INFO (qtp1333520522-2165) [n:127.0.0.1:37555_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1123543 INFO (qtp1333520522-2165) [n:127.0.0.1:37555_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1123543 INFO (qtp1333520522-2165) [n:127.0.0.1:37555_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1593960346290225152
[junit4] 2> 1123571 INFO (searcherExecutor-604-thread-1-processing-n:127.0.0.1:37555_ x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:37555_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@2856b367[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1123579 INFO (qtp1333520522-2165) [n:127.0.0.1:37555_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1123579 INFO (qtp1333520522-2165) [n:127.0.0.1:37555_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1123579 INFO (qtp1333520522-2165) [n:127.0.0.1:37555_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:37555/control_collection_shard1_replica_n1/
[junit4] 2> 1123579 INFO (qtp1333520522-2165) [n:127.0.0.1:37555_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1123579 INFO (qtp1333520522-2165) [n:127.0.0.1:37555_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:37555/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 1123579 INFO (qtp1333520522-2165) [n:127.0.0.1:37555_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 1123582 INFO (qtp1333520522-2165) [n:127.0.0.1:37555_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:37555/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 1123612 INFO (zkCallback-278-thread-1-processing-n:127.0.0.1:37555_) [n:127.0.0.1:37555_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1123662 INFO (qtp1333520522-2165) [n:127.0.0.1:37555_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1123663 INFO (qtp1333520522-2165) [n:127.0.0.1:37555_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2918
[junit4] 2> 1123688 INFO (qtp1333520522-2161) [n:127.0.0.1:37555_ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 1123793 INFO (zkCallback-278-thread-2-processing-n:127.0.0.1:37555_) [n:127.0.0.1:37555_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1124527 INFO (OverseerCollectionConfigSetProcessor-72218579117735940-127.0.0.1:37555_-n_0000000000) [n:127.0.0.1:37555_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1124689 INFO (qtp1333520522-2161) [n:127.0.0.1:37555_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:37555_&wt=javabin&version=2} status=0 QTime=4321
[junit4] 2> 1124742 INFO (zkConnectionManagerCallback-289-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1124743 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1124744 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:43741/solr ready
[junit4] 2> 1124744 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 1124745 INFO (qtp1333520522-2165) [n:127.0.0.1:37555_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1124815 INFO (OverseerThreadFactory-601-thread-2-processing-n:127.0.0.1:37555_) [n:127.0.0.1:37555_ ] o.a.s.c.a.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 1124816 WARN (OverseerThreadFactory-601-thread-2-processing-n:127.0.0.1:37555_) [n:127.0.0.1:37555_ ] o.a.s.c.a.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 1125182 INFO (qtp1333520522-2165) [n:127.0.0.1:37555_ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 1125182 INFO (qtp1333520522-2165) [n:127.0.0.1:37555_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=437
[junit4] 2> 1125826 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.api.collections.ShardSplitTest_CA8C715A4ADC7334-001/shard-1-001 of type TLOG
[junit4] 2> 1125860 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T18:27:37-03:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 1125951 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 1125951 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 1125951 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.e.j.s.session Scavenging every 600000ms
[junit4] 2> 1125951 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@247df2b2{/,null,AVAILABLE}
[junit4] 2> 1125952 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@773671fa{HTTP/1.1,[http/1.1]}{127.0.0.1:38246}
[junit4] 2> 1125952 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.e.j.s.Server Started @1139731ms
[junit4] 2> 1125952 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.api.collections.ShardSplitTest_CA8C715A4ADC7334-001/tempDir-001/jetty1, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/, hostPort=38246, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.api.collections.ShardSplitTest_CA8C715A4ADC7334-001/shard-1-001/cores}
[junit4] 2> 1125952 ERROR (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1125952 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.0.0
[junit4] 2> 1125952 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1125952 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1125952 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-03-03T23:17:26.036Z
[junit4] 2> 1126006 INFO (zkConnectionManagerCallback-291-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1126011 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1126011 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.api.collections.ShardSplitTest_CA8C715A4ADC7334-001/shard-1-001/solr.xml
[junit4] 2> 1126047 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 1126047 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 1126048 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@f1b3f82, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1126083 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:43741/solr
[junit4] 2> 1126172 INFO (zkConnectionManagerCallback-295-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1126282 INFO (zkConnectionManagerCallback-297-thread-1-processing-n:127.0.0.1:38246_) [n:127.0.0.1:38246_ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1126321 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [n:127.0.0.1:38246_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1126355 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [n:127.0.0.1:38246_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1126357 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [n:127.0.0.1:38246_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38246_
[junit4] 2> 1126367 INFO (zkCallback-288-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1126391 INFO (zkCallback-278-thread-2-processing-n:127.0.0.1:37555_) [n:127.0.0.1:37555_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1126392 INFO (zkCallback-296-thread-1-processing-n:127.0.0.1:38246_) [n:127.0.0.1:38246_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1126717 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [n:127.0.0.1:38246_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@f1b3f82
[junit4] 2> 1126790 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [n:127.0.0.1:38246_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@f1b3f82
[junit4] 2> 1126790 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [n:127.0.0.1:38246_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@f1b3f82
[junit4] 2> 1126792 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [n:127.0.0.1:38246_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.api.collections.ShardSplitTest_CA8C715A4ADC7334-001/shard-1-001/cores
[junit4] 2> 1126843 INFO (OverseerCollectionConfigSetProcessor-72218579117735940-127.0.0.1:37555_-n_0000000000) [n:127.0.0.1:37555_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1127011 INFO (qtp1333520522-2163) [n:127.0.0.1:37555_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:38246_&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1127064 INFO (OverseerThreadFactory-601-thread-3-processing-n:127.0.0.1:37555_) [n:127.0.0.1:37555_ ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:38246_ for creating new replica
[junit4] 2> 1127095 INFO (qtp1709580638-2210) [n:127.0.0.1:38246_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 1127096 INFO (qtp1709580638-2210) [n:127.0.0.1:38246_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1128316 INFO (qtp1709580638-2210) [n:127.0.0.1:38246_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 1128456 INFO (qtp1709580638-2210) [n:127.0.0.1:38246_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.s.IndexSchema [collection1_shard2_replica_t41] Schema name=test
[junit4] 2> 1128822 INFO (qtp1709580638-2210) [n:127.0.0.1:38246_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 1128916 INFO (qtp1709580638-2210) [n:127.0.0.1:38246_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_t41' using configuration from collection collection1, trusted=true
[junit4] 2> 1128917 INFO (qtp1709580638-2210) [n:127.0.0.1:38246_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_t41' (registry 'solr.core.collection1.shard2.replica_t41') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@f1b3f82
[junit4] 2> 1128917 INFO (qtp1709580638-2210) [n:127.0.0.1:38246_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1128917 INFO (qtp1709580638-2210) [n:127.0.0.1:38246_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SolrCore [[collection1_shard2_replica_t41] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.api.collections.ShardSplitTest_CA8C715A4ADC7334-001/shard-1-001/cores/collection1_shard2_replica_t41], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.api.collections.ShardSplitTest_CA8C715A4ADC7334-001/shard-1-001/cores/collection1_shard2_replica_t41/data/]
[junit4] 2> 1128936 INFO (qtp1709580638-2210) [n:127.0.0.1:38246_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=24, maxMergeAtOnceExplicit=14, maxMergedSegmentMB=60.7900390625, floorSegmentMB=1.7802734375, forceMergeDeletesPctAllowed=1.0174268134711073, segmentsPerTier=46.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
[junit4] 2> 1128963 WARN (qtp1709580638-2210) [n:127.0.0.1:38246_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1129223 INFO (qtp1709580638-2210) [n:127.0.0.1:38246_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1129223 INFO (qtp1709580638-2210) [n:127.0.0.1:38246_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1129224 INFO (qtp1709580638-2210) [n:127.0.0.1:38246_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1129224 INFO (qtp1709580638-2210) [n:127.0.0.1:38246_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1129226 INFO (qtp1709580638-2210) [n:127.0.0.1:38246_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@3b6fd38
[junit4] 2> 1129226 INFO (qtp1709580638-2210) [n:127.0.0.1:38246_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.s.SolrIndexSearcher Opening [Searcher@64bab395[collection1_shard2_replica_t41] main]
[junit4] 2> 1129260 INFO (qtp1709580638-2210) [n:127.0.0.1:38246_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1129260 INFO (qtp1709580638-2210) [n:127.0.0.1:38246_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1129261 INFO (qtp1709580638-2210) [n:127.0.0.1:38246_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1129261 INFO (qtp1709580638-2210) [n:127.0.0.1:38246_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1593960352285982720
[junit4] 2> 1129281 INFO (searcherExecutor-615-thread-1-processing-n:127.0.0.1:38246_ x:collection1_shard2_replica_t41 s:shard2 c:collection1) [n:127.0.0.1:38246_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SolrCore [collection1_shard2_replica_t41] Registered new searcher Searcher@64bab395[collection1_shard2_replica_t41] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1129313 INFO (qtp1709580638-2210) [n:127.0.0.1:38246_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1129313 INFO (qtp1709580638-2210) [n:127.0.0.1:38246_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1129314 INFO (qtp1709580638-2210) [n:127.0.0.1:38246_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:38246/collection1_shard2_replica_t41/
[junit4] 2> 1129314 INFO (qtp1709580638-2210) [n:127.0.0.1:38246_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1129314 INFO (qtp1709580638-2210) [n:127.0.0.1:38246_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SyncStrategy http://127.0.0.1:38246/collection1_shard2_replica_t41/ has no replicas
[junit4] 2> 1129314 INFO (qtp1709580638-2210) [n:127.0.0.1:38246_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 1129314 INFO (qtp1709580638-2210) [n:127.0.0.1:38246_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ZkController collection1_shard2_replica_t41 stopping background replication from leader
[junit4] 2> 1129333 INFO (qtp1709580638-2210) [n:127.0.0.1:38246_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:38246/collection1_shard2_replica_t41/ shard2
[junit4] 2> 1129353 INFO (qtp1709580638-2210) [n:127.0.0.1:38246_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1129356 INFO (qtp1709580638-2210) [n:127.0.0.1:38246_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=2260
[junit4] 2> 1129390 INFO (qtp1333520522-2163) [n:127.0.0.1:37555_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:38246_&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2} status=0 QTime=2379
[junit4] 2> 1130018 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.api.collections.ShardSplitTest_CA8C715A4ADC7334-001/shard-2-001 of type TLOG
[junit4] 2> 1130018 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T18:27:37-03:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 1130105 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 1130105 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 1130105 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.e.j.s.session Scavenging every 600000ms
[junit4] 2> 1130106 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5ee05e99{/,null,AVAILABLE}
[junit4] 2> 1130106 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@7aa0376f{HTTP/1.1,[http/1.1]}{127.0.0.1:43425}
[junit4] 2> 1130106 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.e.j.s.Server Started @1143885ms
[junit4] 2> 1130106 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.api.collections.ShardSplitTest_CA8C715A4ADC7334-001/tempDir-001/jetty2, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/, hostPort=43425, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.api.collections.ShardSplitTest_CA8C715A4ADC7334-001/shard-2-001/cores}
[junit4] 2> 1130107 ERROR (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1130113 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.0.0
[junit4] 2> 1130113 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1130113 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1130113 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-03-03T23:17:30.197Z
[junit4] 2> 1130149 INFO (zkConnectionManagerCallback-301-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1130151 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1130151 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.api.collections.ShardSplitTest_CA8C715A4ADC7334-001/shard-2-001/solr.xml
[junit4] 2> 1130167 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 1130167 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 1130169 INFO (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[CA8C715A4ADC7334]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@f1b3f82, but no JMX
[...truncated too long message...]
rent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 5) Thread[id=11289, name=searcherExecutor-1564-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 6) Thread[id=11055, name=searcherExecutor-1492-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 7) Thread[id=11231, name=searcherExecutor-1542-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > at __randomizedtesting.SeedInfo.seed([CA8C715A4ADC7334]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
[junit4] > 1) Thread[id=11260, name=searcherExecutor-1553-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 2) Thread[id=11159, name=searcherExecutor-1525-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 3) Thread[id=11091, name=searcherExecutor-1503-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 4) Thread[id=11122, name=searcherExecutor-1514-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 5) Thread[id=11289, name=searcherExecutor-1564-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 6) Thread[id=11055, name=searcherExecutor-1492-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 7) Thread[id=11231, name=searcherExecutor-1542-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > at __randomizedtesting.SeedInfo.seed([CA8C715A4ADC7334]:0)
[junit4] Completed [147/779 (3!)] on J1 in 474.56s, 1 test, 1 failure, 2 errors <<< FAILURES!
[...truncated 49771 lines...]
[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1492 - Still
Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1492/
1 tests failed.
FAILED: org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test
Error Message:
expected:<0> but was:<1>
Stack Trace:
java.lang.AssertionError: expected:<0> but was:<1>
at __randomizedtesting.SeedInfo.seed([DA1F49C466B7CE27:524B761EC84BA3DF]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:472)
at org.junit.Assert.assertEquals(Assert.java:456)
at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test(ChaosMonkeySafeLeaderTest.java:146)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 13264 lines...]
[junit4] Suite: org.apache.solr.cloud.ChaosMonkeySafeLeaderTest
[junit4] 2> 1194671 INFO (SUITE-ChaosMonkeySafeLeaderTest-seed#[DA1F49C466B7CE27]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_DA1F49C466B7CE27-001/init-core-data-001
[junit4] 2> 1194671 WARN (SUITE-ChaosMonkeySafeLeaderTest-seed#[DA1F49C466B7CE27]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=22 numCloses=22
[junit4] 2> 1194671 INFO (SUITE-ChaosMonkeySafeLeaderTest-seed#[DA1F49C466B7CE27]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 1194673 INFO (SUITE-ChaosMonkeySafeLeaderTest-seed#[DA1F49C466B7CE27]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 1194673 INFO (SUITE-ChaosMonkeySafeLeaderTest-seed#[DA1F49C466B7CE27]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
[junit4] 2> 1194678 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 1194678 INFO (Thread-20147) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1194678 INFO (Thread-20147) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 1194679 ERROR (Thread-20147) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 1194778 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.c.ZkTestServer start zk server on port:35256
[junit4] 2> 1194780 INFO (zkConnectionManagerCallback-1764-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1194782 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x104b6b33d6f0000, likely client has closed socket
[junit4] 2> 1194783 INFO (zkConnectionManagerCallback-1766-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1194788 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 1194790 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4] 2> 1194792 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 1194793 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 1194794 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 1194794 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 1194795 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 1194796 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 1194797 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 1194798 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 1194799 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 1194801 INFO (zkConnectionManagerCallback-1769-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1194802 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
[junit4] 2> 1194891 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: ????-??-??T??:??:??+08:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 1194892 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 1194892 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 1194892 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.e.j.s.session Scavenging every 600000ms
[junit4] 2> 1194892 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6721712f{/,null,AVAILABLE}
[junit4] 2> 1194894 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@4f2aca17{SSL,[ssl, http/1.1]}{127.0.0.1:39919}
[junit4] 2> 1194894 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.e.j.s.Server Started @1197040ms
[junit4] 2> 1194894 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_DA1F49C466B7CE27-001/tempDir-001/control/data, replicaType=NRT, hostContext=/, hostPort=39919, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_DA1F49C466B7CE27-001/control-001/cores}
[junit4] 2> 1194894 ERROR (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1194895 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 8.0.0
[junit4] 2> 1194895 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1194895 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1194895 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-03-02T12:19:23.704Z
[junit4] 2> 1194901 INFO (zkConnectionManagerCallback-1771-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1194902 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1194902 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_DA1F49C466B7CE27-001/control-001/solr.xml
[junit4] 2> 1194905 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 1194905 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 1194906 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@2830f932, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1194909 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:35256/solr
[junit4] 2> 1194916 INFO (zkConnectionManagerCallback-1775-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1194917 INFO (zkConnectionManagerCallback-1777-thread-1-processing-n:127.0.0.1:39919_) [n:127.0.0.1:39919_ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1194976 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [n:127.0.0.1:39919_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1194976 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [n:127.0.0.1:39919_ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:39919_
[junit4] 2> 1194977 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [n:127.0.0.1:39919_ ] o.a.s.c.Overseer Overseer (id=73384374890856453-127.0.0.1:39919_-n_0000000000) starting
[junit4] 2> 1194982 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [n:127.0.0.1:39919_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:39919_
[junit4] 2> 1194987 INFO (OverseerStateUpdate-73384374890856453-127.0.0.1:39919_-n_0000000000) [n:127.0.0.1:39919_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1195046 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [n:127.0.0.1:39919_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2830f932
[junit4] 2> 1195054 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [n:127.0.0.1:39919_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2830f932
[junit4] 2> 1195054 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [n:127.0.0.1:39919_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2830f932
[junit4] 2> 1195056 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [n:127.0.0.1:39919_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_DA1F49C466B7CE27-001/control-001/cores
[junit4] 2> 1195073 INFO (zkConnectionManagerCallback-1783-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1195074 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1195075 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:35256/solr ready
[junit4] 2> 1195090 INFO (qtp2030546090-26717) [n:127.0.0.1:39919_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:39919_&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1195092 INFO (OverseerThreadFactory-4247-thread-1-processing-n:127.0.0.1:39919_) [n:127.0.0.1:39919_ ] o.a.s.c.a.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 1195199 INFO (qtp2030546090-26718) [n:127.0.0.1:39919_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 1195199 INFO (qtp2030546090-26718) [n:127.0.0.1:39919_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1195303 INFO (zkCallback-1776-thread-1-processing-n:127.0.0.1:39919_) [n:127.0.0.1:39919_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1196212 INFO (qtp2030546090-26718) [n:127.0.0.1:39919_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 1196225 INFO (qtp2030546090-26718) [n:127.0.0.1:39919_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 1196318 INFO (qtp2030546090-26718) [n:127.0.0.1:39919_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 1196329 INFO (qtp2030546090-26718) [n:127.0.0.1:39919_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 1196329 INFO (qtp2030546090-26718) [n:127.0.0.1:39919_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2830f932
[junit4] 2> 1196330 INFO (qtp2030546090-26718) [n:127.0.0.1:39919_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1196330 INFO (qtp2030546090-26718) [n:127.0.0.1:39919_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_DA1F49C466B7CE27-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_DA1F49C466B7CE27-001/control-001/cores/control_collection_shard1_replica_n1/data/]
[junit4] 2> 1196332 INFO (qtp2030546090-26718) [n:127.0.0.1:39919_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=27, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 1196341 WARN (qtp2030546090-26718) [n:127.0.0.1:39919_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1196375 INFO (qtp2030546090-26718) [n:127.0.0.1:39919_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1196375 INFO (qtp2030546090-26718) [n:127.0.0.1:39919_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1196376 INFO (qtp2030546090-26718) [n:127.0.0.1:39919_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 1196376 INFO (qtp2030546090-26718) [n:127.0.0.1:39919_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1196378 INFO (qtp2030546090-26718) [n:127.0.0.1:39919_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=43, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 1196378 INFO (qtp2030546090-26718) [n:127.0.0.1:39919_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@635e5191[control_collection_shard1_replica_n1] main]
[junit4] 2> 1196379 INFO (qtp2030546090-26718) [n:127.0.0.1:39919_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1196379 INFO (qtp2030546090-26718) [n:127.0.0.1:39919_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1196380 INFO (qtp2030546090-26718) [n:127.0.0.1:39919_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1196381 INFO (searcherExecutor-4250-thread-1-processing-n:127.0.0.1:39919_ x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:39919_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@635e5191[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1196381 INFO (qtp2030546090-26718) [n:127.0.0.1:39919_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1593828353182269440
[junit4] 2> 1196387 INFO (qtp2030546090-26718) [n:127.0.0.1:39919_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1196387 INFO (qtp2030546090-26718) [n:127.0.0.1:39919_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1196387 INFO (qtp2030546090-26718) [n:127.0.0.1:39919_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:39919/control_collection_shard1_replica_n1/
[junit4] 2> 1196387 INFO (qtp2030546090-26718) [n:127.0.0.1:39919_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1196387 INFO (qtp2030546090-26718) [n:127.0.0.1:39919_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy https://127.0.0.1:39919/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 1196387 INFO (qtp2030546090-26718) [n:127.0.0.1:39919_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 1196389 INFO (qtp2030546090-26718) [n:127.0.0.1:39919_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:39919/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 1196490 INFO (zkCallback-1776-thread-1-processing-n:127.0.0.1:39919_) [n:127.0.0.1:39919_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1196540 INFO (qtp2030546090-26718) [n:127.0.0.1:39919_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1196542 INFO (qtp2030546090-26718) [n:127.0.0.1:39919_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1342
[junit4] 2> 1196544 INFO (qtp2030546090-26717) [n:127.0.0.1:39919_ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 1196642 INFO (zkCallback-1776-thread-2-processing-n:127.0.0.1:39919_) [n:127.0.0.1:39919_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1197094 INFO (OverseerCollectionConfigSetProcessor-73384374890856453-127.0.0.1:39919_-n_0000000000) [n:127.0.0.1:39919_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1197544 INFO (qtp2030546090-26717) [n:127.0.0.1:39919_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:39919_&wt=javabin&version=2} status=0 QTime=2453
[junit4] 2> 1197548 INFO (zkConnectionManagerCallback-1787-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1197549 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1197550 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:35256/solr ready
[junit4] 2> 1197550 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 1197552 INFO (qtp2030546090-26717) [n:127.0.0.1:39919_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=4&createNodeSet=&stateFormat=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1197553 INFO (OverseerThreadFactory-4247-thread-2-processing-n:127.0.0.1:39919_) [n:127.0.0.1:39919_ ] o.a.s.c.a.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 1197554 WARN (OverseerThreadFactory-4247-thread-2-processing-n:127.0.0.1:39919_) [n:127.0.0.1:39919_ ] o.a.s.c.a.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 1197757 INFO (qtp2030546090-26717) [n:127.0.0.1:39919_ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 1197758 INFO (qtp2030546090-26717) [n:127.0.0.1:39919_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=4&createNodeSet=&stateFormat=2&wt=javabin&version=2} status=0 QTime=206
[junit4] 2> 1197852 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_DA1F49C466B7CE27-001/shard-1-001 of type NRT
[junit4] 2> 1197854 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: ????-??-??T??:??:??+08:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 1197858 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 1197858 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 1197859 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.e.j.s.session Scavenging every 660000ms
[junit4] 2> 1197859 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4876d8a6{/,null,AVAILABLE}
[junit4] 2> 1197860 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@43ca5705{SSL,[ssl, http/1.1]}{127.0.0.1:48762}
[junit4] 2> 1197860 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.e.j.s.Server Started @1200006ms
[junit4] 2> 1197860 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_DA1F49C466B7CE27-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/, hostPort=48762, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_DA1F49C466B7CE27-001/shard-1-001/cores}
[junit4] 2> 1197860 ERROR (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1197860 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 8.0.0
[junit4] 2> 1197860 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1197861 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1197861 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-03-02T12:19:26.670Z
[junit4] 2> 1197863 INFO (zkConnectionManagerCallback-1789-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1197866 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1197866 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_DA1F49C466B7CE27-001/shard-1-001/solr.xml
[junit4] 2> 1197869 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 1197869 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 1197871 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@2830f932, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1197874 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:35256/solr
[junit4] 2> 1197887 INFO (zkConnectionManagerCallback-1793-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1197889 INFO (zkConnectionManagerCallback-1795-thread-1-processing-n:127.0.0.1:48762_) [n:127.0.0.1:48762_ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1197895 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [n:127.0.0.1:48762_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1197897 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [n:127.0.0.1:48762_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1197898 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [n:127.0.0.1:48762_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:48762_
[junit4] 2> 1197899 INFO (zkCallback-1776-thread-2-processing-n:127.0.0.1:39919_) [n:127.0.0.1:39919_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1197899 INFO (zkCallback-1786-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1197900 INFO (zkCallback-1794-thread-1-processing-n:127.0.0.1:48762_) [n:127.0.0.1:48762_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1197950 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [n:127.0.0.1:48762_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2830f932
[junit4] 2> 1197973 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [n:127.0.0.1:48762_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2830f932
[junit4] 2> 1197973 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [n:127.0.0.1:48762_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2830f932
[junit4] 2> 1197975 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [n:127.0.0.1:48762_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_DA1F49C466B7CE27-001/shard-1-001/cores
[junit4] 2> 1197999 INFO (qtp2030546090-26715) [n:127.0.0.1:39919_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:48762_&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1198001 INFO (OverseerCollectionConfigSetProcessor-73384374890856453-127.0.0.1:39919_-n_0000000000) [n:127.0.0.1:39919_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1198001 INFO (OverseerThreadFactory-4247-thread-3-processing-n:127.0.0.1:39919_) [n:127.0.0.1:39919_ ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:48762_ for creating new replica
[junit4] 2> 1198020 INFO (qtp883375928-26763) [n:127.0.0.1:48762_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n81&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 1198020 INFO (qtp883375928-26763) [n:127.0.0.1:48762_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1198124 INFO (zkCallback-1794-thread-1-processing-n:127.0.0.1:48762_) [n:127.0.0.1:48762_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1199034 INFO (qtp883375928-26763) [n:127.0.0.1:48762_ c:collection1 s:shard2 x:collection1_shard2_replica_n81] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 1199044 INFO (qtp883375928-26763) [n:127.0.0.1:48762_ c:collection1 s:shard2 x:collection1_shard2_replica_n81] o.a.s.s.IndexSchema [collection1_shard2_replica_n81] Schema name=test
[junit4] 2> 1199125 INFO (qtp883375928-26763) [n:127.0.0.1:48762_ c:collection1 s:shard2 x:collection1_shard2_replica_n81] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 1199136 INFO (qtp883375928-26763) [n:127.0.0.1:48762_ c:collection1 s:shard2 x:collection1_shard2_replica_n81] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_n81' using configuration from collection collection1, trusted=true
[junit4] 2> 1199137 INFO (qtp883375928-26763) [n:127.0.0.1:48762_ c:collection1 s:shard2 x:collection1_shard2_replica_n81] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_n81' (registry 'solr.core.collection1.shard2.replica_n81') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2830f932
[junit4] 2> 1199137 INFO (qtp883375928-26763) [n:127.0.0.1:48762_ c:collection1 s:shard2 x:collection1_shard2_replica_n81] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1199137 INFO (qtp883375928-26763) [n:127.0.0.1:48762_ c:collection1 s:shard2 x:collection1_shard2_replica_n81] o.a.s.c.SolrCore [[collection1_shard2_replica_n81] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_DA1F49C466B7CE27-001/shard-1-001/cores/collection1_shard2_replica_n81], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_DA1F49C466B7CE27-001/shard-1-001/cores/collection1_shard2_replica_n81/data/]
[junit4] 2> 1199140 INFO (qtp883375928-26763) [n:127.0.0.1:48762_ c:collection1 s:shard2 x:collection1_shard2_replica_n81] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=27, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 1199148 WARN (qtp883375928-26763) [n:127.0.0.1:48762_ c:collection1 s:shard2 x:collection1_shard2_replica_n81] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1199186 INFO (qtp883375928-26763) [n:127.0.0.1:48762_ c:collection1 s:shard2 x:collection1_shard2_replica_n81] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1199187 INFO (qtp883375928-26763) [n:127.0.0.1:48762_ c:collection1 s:shard2 x:collection1_shard2_replica_n81] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1199188 INFO (qtp883375928-26763) [n:127.0.0.1:48762_ c:collection1 s:shard2 x:collection1_shard2_replica_n81] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 1199188 INFO (qtp883375928-26763) [n:127.0.0.1:48762_ c:collection1 s:shard2 x:collection1_shard2_replica_n81] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1199189 INFO (qtp883375928-26763) [n:127.0.0.1:48762_ c:collection1 s:shard2 x:collection1_shard2_replica_n81] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=43, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 1199189 INFO (qtp883375928-26763) [n:127.0.0.1:48762_ c:collection1 s:shard2 x:collection1_shard2_replica_n81] o.a.s.s.SolrIndexSearcher Opening [Searcher@44b7dd94[collection1_shard2_replica_n81] main]
[junit4] 2> 1199190 INFO (qtp883375928-26763) [n:127.0.0.1:48762_ c:collection1 s:shard2 x:collection1_shard2_replica_n81] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1199191 INFO (qtp883375928-26763) [n:127.0.0.1:48762_ c:collection1 s:shard2 x:collection1_shard2_replica_n81] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1199191 INFO (qtp883375928-26763) [n:127.0.0.1:48762_ c:collection1 s:shard2 x:collection1_shard2_replica_n81] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1199192 INFO (searcherExecutor-4261-thread-1-processing-n:127.0.0.1:48762_ x:collection1_shard2_replica_n81 s:shard2 c:collection1) [n:127.0.0.1:48762_ c:collection1 s:shard2 x:collection1_shard2_replica_n81] o.a.s.c.SolrCore [collection1_shard2_replica_n81] Registered new searcher Searcher@44b7dd94[collection1_shard2_replica_n81] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1199192 INFO (qtp883375928-26763) [n:127.0.0.1:48762_ c:collection1 s:shard2 x:collection1_shard2_replica_n81] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1593828356129816576
[junit4] 2> 1199199 INFO (qtp883375928-26763) [n:127.0.0.1:48762_ c:collection1 s:shard2 x:collection1_shard2_replica_n81] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1199199 INFO (qtp883375928-26763) [n:127.0.0.1:48762_ c:collection1 s:shard2 x:collection1_shard2_replica_n81] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1199199 INFO (qtp883375928-26763) [n:127.0.0.1:48762_ c:collection1 s:shard2 x:collection1_shard2_replica_n81] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:48762/collection1_shard2_replica_n81/
[junit4] 2> 1199199 INFO (qtp883375928-26763) [n:127.0.0.1:48762_ c:collection1 s:shard2 x:collection1_shard2_replica_n81] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1199199 INFO (qtp883375928-26763) [n:127.0.0.1:48762_ c:collection1 s:shard2 x:collection1_shard2_replica_n81] o.a.s.c.SyncStrategy https://127.0.0.1:48762/collection1_shard2_replica_n81/ has no replicas
[junit4] 2> 1199199 INFO (qtp883375928-26763) [n:127.0.0.1:48762_ c:collection1 s:shard2 x:collection1_shard2_replica_n81] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 1199202 INFO (qtp883375928-26763) [n:127.0.0.1:48762_ c:collection1 s:shard2 x:collection1_shard2_replica_n81] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:48762/collection1_shard2_replica_n81/ shard2
[junit4] 2> 1199303 INFO (zkCallback-1794-thread-1-processing-n:127.0.0.1:48762_) [n:127.0.0.1:48762_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1199352 INFO (qtp883375928-26763) [n:127.0.0.1:48762_ c:collection1 s:shard2 x:collection1_shard2_replica_n81] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1199354 INFO (qtp883375928-26763) [n:127.0.0.1:48762_ c:collection1 s:shard2 x:collection1_shard2_replica_n81] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n81&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1334
[junit4] 2> 1199356 INFO (qtp2030546090-26715) [n:127.0.0.1:39919_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:48762_&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2} status=0 QTime=1357
[junit4] 2> 1199454 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_DA1F49C466B7CE27-001/shard-2-001 of type NRT
[junit4] 2> 1199455 INFO (zkCallback-1794-thread-1-processing-n:127.0.0.1:48762_) [n:127.0.0.1:48762_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1199457 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: ????-??-??T??:??:??+08:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 1199458 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 1199458 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 1199458 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.e.j.s.session Scavenging every 600000ms
[junit4] 2> 1199458 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3850970c{/,null,AVAILABLE}
[junit4] 2> 1199459 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@7b1010{SSL,[ssl, http/1.1]}{127.0.0.1:59794}
[junit4] 2> 1199459 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.e.j.s.Server Started @1201605ms
[junit4] 2> 1199459 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_DA1F49C466B7CE27-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostContext=/, hostPort=59794, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_DA1F49C466B7CE27-001/shard-2-001/cores}
[junit4] 2> 1199459 ERROR (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1199459 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 8.0.0
[junit4] 2> 1199459 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1199459 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1199459 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-03-02T12:19:28.268Z
[junit4] 2> 1199461 INFO (zkConnectionManagerCallback-1799-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1199461 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1199462 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_DA1F49C466B7CE27-001/shard-2-001/solr.xml
[junit4] 2> 1199465 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 1199465 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 1199466 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@2830f932, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1199468 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:35256/solr
[junit4] 2> 1199469 INFO (zkConnectionManagerCallback-1803-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1199470 INFO (zkConnectionManagerCallback-1805-thread-1-processing-n:127.0.0.1:59794_) [n:127.0.0.1:59794_ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1199475 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [n:127.0.0.1:59794_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 1199476 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [n:127.0.0.1:59794_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1199478 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [n:127.0.0.1:59794_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:59794_
[junit4] 2> 1199479 INFO (zkCallback-1794-thread-1-processing-n:127.0.0.1:48762_) [n:127.0.0.1:48762_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1199479 INFO (zkCallback-1776-thread-2-processing-n:127.0.0.1:39919_) [n:127.0.0.1:39919_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1199479 INFO (zkCallback-1786-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1199480 INFO (zkCallback-1804-thread-1-processing-n:127.0.0.1:59794_) [n:127.0.0.1:59794_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1199551 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [n:127.0.0.1:59794_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2830f932
[junit4] 2> 1199560 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [n:127.0.0.1:59794_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2830f932
[junit4] 2> 1199560 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [n:127.0.0.1:59794_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2830f932
[junit4] 2> 1199562 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [n:127.0.0.1:59794_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_DA1F49C466B7CE27-001/shard-2-001/cores
[junit4] 2> 1199594 INFO (qtp2030546090-26717) [n:127.0.0.1:39919_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:59794_&action=ADDREPLICA&collection=collection1&shard=shard3&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1199596 INFO (OverseerCollectionConfigSetProcessor-73384374890856453-127.0.0.1:39919_-n_0000000000) [n:127.0.0.1:39919_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1199596 INFO (OverseerThreadFactory-4247-thread-4-processing-n:127.0.0.1:39919_) [n:127.0.0.1:39919_ ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:59794_ for creating new replica
[junit4] 2> 1199615 INFO (qtp792556971-26795) [n:127.0.0.1:59794_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard3_replica_n83&action=CREATE&collection=collection1&shard=shard3&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 1199615 INFO (qtp792556971-26795) [n:127.0.0.1:59794_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1199719 INFO (zkCallback-1794-thread-1-processing-n:127.0.0.1:48762_) [n:127.0.0.1:48762_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 1199719 INFO (zkCallback-1804-thread-1-processing-n:127.0.0.1:59794_) [n:127.0.0.1:59794_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 1200628 INFO (qtp792556971-26795) [n:127.0.0.1:59794_ c:collection1 s:shard3 x:collection1_shard3_replica_n83] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 1200641 INFO (qtp792556971-26795) [n:127.0.0.1:59794_ c:collection1 s:shard3 x:collection1_shard3_replica_n83] o.a.s.s.IndexSchema [collection1_shard3_replica_n83] Schema name=test
[junit4] 2> 1200731 INFO (qtp792556971-26795) [n:127.0.0.1:59794_ c:collection1 s:shard3 x:collection1_shard3_replica_n83] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 1200742 INFO (qtp792556971-26795) [n:127.0.0.1:59794_ c:collection1 s:shard3 x:collection1_shard3_replica_n83] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard3_replica_n83' using configuration from collection collection1, trusted=true
[junit4] 2> 1200743 INFO (qtp792556971-26795) [n:127.0.0.1:59794_ c:collection1 s:shard3 x:collection1_shard3_replica_n83] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard3.replica_n83' (registry 'solr.core.collection1.shard3.replica_n83') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2830f932
[junit4] 2> 1200743 INFO (qtp792556971-26795) [n:127.0.0.1:59794_ c:collection1 s:shard3 x:collection1_shard3_replica_n83] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1200743 INFO (qtp792556971-26795) [n:127.0.0.1:59794_ c:collection1 s:shard3 x:collection1_shard3_replica_n83] o.a.s.c.SolrCore [[collection1_shard3_replica_n83] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_DA1F49C466B7CE27-001/shard-2-001/cores/collection1_shard3_replica_n83], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_DA1F49C466B7CE27-001/shard-2-001/cores/collection1_shard3_replica_n83/data/]
[junit4] 2> 1200746 INFO (qtp792556971-26795) [n:127.0.0.1:59794_ c:collection1 s:shard3 x:collection1_shard3_replica_n83] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=27, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 1200754 WARN (qtp792556971-26795) [n:127.0.0.1:59794_ c:collection1 s:shard3 x:collection1_shard3_replica_n83] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1200790 INFO (qtp792556971-26795) [n:127.0.0.1:59794_ c:collection1 s:shard3 x:collection1_shard3_replica_n83] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1200790 INFO (qtp792556971-26795) [n:127.0.0.1:59794_ c:collection1 s:shard3 x:collection1_shard3_replica_n83] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1200791 INFO (qtp792556971-26795) [n:127.0.0.1:59794_ c:collection1 s:shard3 x:collection1_shard3_replica_n83] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 1200791 INFO (qtp792556971-26795) [n:127.0.0.1:59794_ c:collection1 s:shard3 x:collection1_shard3_replica_n83] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1200792 INFO (qtp792556971-26795) [n:127.0.0.1:59794_ c:collection1 s:shard3 x:collection1_shard3_replica_n83] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=43, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 1200792 INFO (qtp792556971-26795) [n:127.0.0.1:59794_ c:collection1 s:shard3 x:collection1_shard3_replica_n83] o.a.s.s.SolrIndexSearcher Opening [Searcher@64acd7ba[collection1_shard3_replica_n83] main]
[junit4] 2> 1200793 INFO (qtp792556971-26795) [n:127.0.0.1:59794_ c:collection1 s:shard3 x:collection1_shard3_replica_n83] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1200793 INFO (qtp792556971-26795) [n:127.0.0.1:59794_ c:collection1 s:shard3 x:collection1_shard3_replica_n83] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1200794 INFO (qtp792556971-26795) [n:127.0.0.1:59794_ c:collection1 s:shard3 x:collection1_shard3_replica_n83] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1200795 INFO (searcherExecutor-4272-thread-1-processing-n:127.0.0.1:59794_ x:collection1_shard3_replica_n83 s:shard3 c:collection1) [n:127.0.0.1:59794_ c:collection1 s:shard3 x:collection1_shard3_replica_n83] o.a.s.c.SolrCore [collection1_shard3_replica_n83] Registered new searcher Searcher@64acd7ba[collection1_shard3_replica_n83] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1200795 INFO (qtp792556971-26795) [n:127.0.0.1:59794_ c:collection1 s:shard3 x:collection1_shard3_replica_n83] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1593828357810683904
[junit4] 2> 1200800 INFO (qtp792556971-26795) [n:127.0.0.1:59794_ c:collection1 s:shard3 x:collection1_shard3_replica_n83] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1200800 INFO (qtp792556971-26795) [n:127.0.0.1:59794_ c:collection1 s:shard3 x:collection1_shard3_replica_n83] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1200800 INFO (qtp792556971-26795) [n:127.0.0.1:59794_ c:collection1 s:shard3 x:collection1_shard3_replica_n83] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:59794/collection1_shard3_replica_n83/
[junit4] 2> 1200800 INFO (qtp792556971-26795) [n:127.0.0.1:59794_ c:collection1 s:shard3 x:collection1_shard3_replica_n83] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1200800 INFO (qtp792556971-26795) [n:127.0.0.1:59794_ c:collection1 s:shard3 x:collection1_shard3_replica_n83] o.a.s.c.SyncStrategy https://127.0.0.1:59794/collection1_shard3_replica_n83/ has no replicas
[junit4] 2> 1200800 INFO (qtp792556971-26795) [n:127.0.0.1:59794_ c:collection1 s:shard3 x:collection1_shard3_replica_n83] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 1200803 INFO (qtp792556971-26795) [n:127.0.0.1:59794_ c:collection1 s:shard3 x:collection1_shard3_replica_n83] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:59794/collection1_shard3_replica_n83/ shard3
[junit4] 2> 1200904 INFO (zkCallback-1794-thread-1-processing-n:127.0.0.1:48762_) [n:127.0.0.1:48762_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 1200904 INFO (zkCallback-1804-thread-1-processing-n:127.0.0.1:59794_) [n:127.0.0.1:59794_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 1200954 INFO (qtp792556971-26795) [n:127.0.0.1:59794_ c:collection1 s:shard3 x:collection1_shard3_replica_n83] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1200955 INFO (qtp792556971-26795) [n:127.0.0.1:59794_ c:collection1 s:shard3 x:collection1_shard3_replica_n83] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard3_replica_n83&action=CREATE&collection=collection1&shard=shard3&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1340
[junit4] 2> 1200957 INFO (qtp2030546090-26717) [n:127.0.0.1:39919_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:59794_&action=ADDREPLICA&collection=collection1&shard=shard3&type=NRT&wt=javabin&version=2} status=0 QTime=1363
[junit4] 2> 1201056 INFO (zkCallback-1794-thread-1-processing-n:127.0.0.1:48762_) [n:127.0.0.1:48762_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 1201056 INFO (zkCallback-1804-thread-1-processing-n:127.0.0.1:59794_) [n:127.0.0.1:59794_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 1201062 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_DA1F49C466B7CE27-001/shard-3-001 of type NRT
[junit4] 2> 1201064 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: ????-??-??T??:??:??+08:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 1201065 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 1201065 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 1201065 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.e.j.s.session Scavenging every 660000ms
[junit4] 2> 1201065 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2d41ace2{/,null,AVAILABLE}
[junit4] 2> 1201066 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@12cd0e31{SSL,[ssl, http/1.1]}{127.0.0.1:43742}
[junit4] 2> 1201066 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.e.j.s.Server Started @1203212ms
[junit4] 2> 1201066 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_DA1F49C466B7CE27-001/tempDir-001/jetty3, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/, hostPort=43742, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_DA1F49C466B7CE27-001/shard-3-001/cores}
[junit4] 2> 1201066 ERROR (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1201066 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 8.0.0
[junit4] 2> 1201066 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1201066 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1201066 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-03-02T12:19:29.875Z
[junit4] 2> 1201070 INFO (zkConnectionManagerCallback-1809-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1201078 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1201079 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_DA1F49C466B7CE27-001/shard-3-001/solr.xml
[junit4] 2> 1201082 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 1201082 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 1201083 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@2830f932, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1201085 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:35256/solr
[junit4] 2> 1201086 INFO (zkConnectionManagerCallback-1813-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1201088 INFO (zkConnectionManagerCallback-1815-thread-1-processing-n:127.0.0.1:43742_) [n:127.0.0.1:43742_ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1201093 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [n:127.0.0.1:43742_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 1201095 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [n:127.0.0.1:43742_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1201096 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [n:127.0.0.1:43742_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43742_
[junit4] 2> 1201097 INFO (zkCallback-1776-thread-2-processing-n:127.0.0.1:39919_) [n:127.0.0.1:39919_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1201097 INFO (zkCallback-1794-thread-1-processing-n:127.0.0.1:48762_) [n:127.0.0.1:48762_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1201097 INFO (zkCallback-1804-thread-1-processing-n:127.0.0.1:59794_) [n:127.0.0.1:59794_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1201097 INFO (zkCallback-1786-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1201098 INFO (zkCallback-1814-thread-1-processing-n:127.0.0.1:43742_) [n:127.0.0.1:43742_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1201169 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [n:127.0.0.1:43742_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2830f932
[junit4] 2> 1201179 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [n:127.0.0.1:43742_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2830f932
[junit4] 2> 1201179 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [n:127.0.0.1:43742_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2830f932
[junit4] 2> 1201181 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [n:127.0.0.1:43742_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_DA1F49C466B7CE27-001/shard-3-001/cores
[junit4] 2> 1201224 INFO (qtp2030546090-26715) [n:127.0.0.1:39919_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:43742_&action=ADDREPLICA&collection=collection1&shard=shard4&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1201227 INFO (OverseerCollectionConfigSetProcessor-73384374890856453-127.0.0.1:39919_-n_0000000000) [n:127.0.0.1:39919_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000006 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1201228 INFO (OverseerThreadFactory-4247-thread-5-processing-n:127.0.0.1:39919_) [n:127.0.0.1:39919_ ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:43742_ for creating new replica
[junit4] 2> 1201248 INFO (qtp1633479332-26828) [n:127.0.0.1:43742_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard4_replica_n85&action=CREATE&collection=collection1&shard=shard4&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 1201248 INFO (qtp1633479332-26828) [n:127.0.0.1:43742_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1201352 INFO (zkCallback-1804-thread-1-processing-n:127.0.0.1:59794_) [n:127.0.0.1:59794_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1201352 INFO (zkCallback-1814-thread-1-processing-n:127.0.0.1:43742_) [n:127.0.0.1:43742_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1201352 INFO (zkCallback-1794-thread-1-processing-n:127.0.0.1:48762_) [n:127.0.0.1:48762_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1202261 INFO (qtp1633479332-26828) [n:127.0.0.1:43742_ c:collection1 s:shard4 x:collection1_shard4_replica_n85] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 1202273 INFO (qtp1633479332-26828) [n:127.0.0.1:43742_ c:collection1 s:shard4 x:collection1_shard4_replica_n85] o.a.s.s.IndexSchema [collection1_shard4_replica_n85] Schema name=test
[junit4] 2> 1202360 INFO (qtp1633479332-26828) [n:127.0.0.1:43742_ c:collection1 s:shard4 x:collection1_shard4_replica_n85] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 1202371 INFO (qtp1633479332-26828) [n:127.0.0.1:43742_ c:collection1 s:shard4 x:collection1_shard4_replica_n85] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard4_replica_n85' using configuration from collection collection1, trusted=true
[junit4] 2> 1202372 INFO (qtp1633479332-26828) [n:127.0.0.1:43742_ c:collection1 s:shard4 x:collection1_shard4_replica_n85] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard4.replica_n85' (registry 'solr.core.collection1.shard4.replica_n85') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2830f932
[junit4] 2> 1202372 INFO (qtp1633479332-26828) [n:127.0.0.1:43742_ c:collection1 s:shard4 x:collection1_shard4_replica_n85] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1202372 INFO (qtp1633479332-26828) [n:127.0.0.1:43742_ c:collection1 s:shard4 x:collection1_shard4_replica_n85] o.a.s.c.SolrCore [[collection1_shard4_replica_n85] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_DA1F49C466B7CE27-001/shard-3-001/cores/collection1_shard4_replica_n85], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_DA1F49C466B7CE27-001/shard-3-001/cores/collection1_shard4_replica_n85/data/]
[junit4] 2> 1202375 INFO (qtp1633479332-26828) [n:127.0.0.1:43742_ c:collection1 s:shard4 x:collection1_shard4_replica_n85] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=27, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 1202383 WARN (qtp1633479332-26828) [n:127.0.0.1:43742_ c:collection1 s:shard4 x:collection1_shard4_replica_n85] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1202422 INFO (qtp1633479332-26828) [n:127.0.0.1:43742_ c:collection1 s:shard4 x:collection1_shard4_replica_n85] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1202422 INFO (qtp1633479332-26828) [n:127.0.0.1:43742_ c:collection1 s:shard4 x:collection1_shard4_replica_n85] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1202423 INFO (qtp1633479332-26828) [n:127.0.0.1:43742_ c:collection1 s:shard4 x:collection1_shard4_replica_n85] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 1202423 INFO (qtp1633479332-26828) [n:127.0.0.1:43742_ c:collection1 s:shard4 x:collection1_shard4_replica_n85] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1202424 INFO (qtp1633479332-26828) [n:127.0.0.1:43742_ c:collection1 s:shard4 x:collection1_shard4_replica_n85] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=43, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 1202425 INFO (qtp1633479332-26828) [n:127.0.0.1:43742_ c:collection1 s:shard4 x:collection1_shard4_replica_n85] o.a.s.s.SolrIndexSearcher Opening [Searcher@2f83934d[collection1_shard4_replica_n85] main]
[junit4] 2> 1202426 INFO (qtp1633479332-26828) [n:127.0.0.1:43742_ c:collection1 s:shard4 x:collection1_shard4_replica_n85] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1202426 INFO (qtp1633479332-26828) [n:127.0.0.1:43742_ c:collection1 s:shard4 x:collection1_shard4_replica_n85] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1202427 INFO (qtp1633479332-26828) [n:127.0.0.1:43742_ c:collection1 s:shard4 x:collection1_shard4_replica_n85] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1202428 INFO (searcherExecutor-4283-thread-1-processing-n:127.0.0.1:43742_ x:collection1_shard4_replica_n85 s:shard4 c:collection1) [n:127.0.0.1:43742_ c:collection1 s:shard4 x:collection1_shard4_replica_n85] o.a.s.c.SolrCore [collection1_shard4_replica_n85] Registered new searcher Searcher@2f83934d[collection1_shard4_replica_n85] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1202428 INFO (qtp1633479332-26828) [n:127.0.0.1:43742_ c:collection1 s:shard4 x:collection1_shard4_replica_n85] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1593828359523008512
[junit4] 2> 1202434 INFO (qtp1633479332-26828) [n:127.0.0.1:43742_ c:collection1 s:shard4 x:collection1_shard4_replica_n85] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1202434 INFO (qtp1633479332-26828) [n:127.0.0.1:43742_ c:collection1 s:shard4 x:collection1_shard4_replica_n85] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1202434 INFO (qtp1633479332-26828) [n:127.0.0.1:43742_ c:collection1 s:shard4 x:collection1_shard4_replica_n85] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:43742/collection1_shard4_replica_n85/
[junit4] 2> 1202434 INFO (qtp1633479332-26828) [n:127.0.0.1:43742_ c:collection1 s:shard4 x:collection1_shard4_replica_n85] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1202434 INFO (qtp1633479332-26828) [n:127.0.0.1:43742_ c:collection1 s:shard4 x:collection1_shard4_replica_n85] o.a.s.c.SyncStrategy https://127.0.0.1:43742/collection1_shard4_replica_n85/ has no replicas
[junit4] 2> 1202434 INFO (qtp1633479332-26828) [n:127.0.0.1:43742_ c:collection1 s:shard4 x:collection1_shard4_replica_n85] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 1202436 INFO (qtp1633479332-26828) [n:127.0.0.1:43742_ c:collection1 s:shard4 x:collection1_shard4_replica_n85] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:43742/collection1_shard4_replica_n85/ shard4
[junit4] 2> 1202538 INFO (zkCallback-1814-thread-1-processing-n:127.0.0.1:43742_) [n:127.0.0.1:43742_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1202538 INFO (zkCallback-1804-thread-1-processing-n:127.0.0.1:59794_) [n:127.0.0.1:59794_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1202538 INFO (zkCallback-1794-thread-1-processing-n:127.0.0.1:48762_) [n:127.0.0.1:48762_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1202587 INFO (qtp1633479332-26828) [n:127.0.0.1:43742_ c:collection1 s:shard4 x:collection1_shard4_replica_n85] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1202589 INFO (qtp1633479332-26828) [n:127.0.0.1:43742_ c:collection1 s:shard4 x:collection1_shard4_replica_n85] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard4_replica_n85&action=CREATE&collection=collection1&shard=shard4&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1340
[junit4] 2> 1202591 INFO (qtp2030546090-26715) [n:127.0.0.1:39919_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:43742_&action=ADDREPLICA&collection=collection1&shard=shard4&type=NRT&wt=javabin&version=2} status=0 QTime=1366
[junit4] 2> 1202681 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 4 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_DA1F49C466B7CE27-001/shard-4-001 of type NRT
[junit4] 2> 1202682 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: ????-??-??T??:??:??+08:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 1202683 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 1202683 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 1202683 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.e.j.s.session Scavenging every 660000ms
[junit4] 2> 1202683 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@52523738{/,null,AVAILABLE}
[junit4] 2> 1202684 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@4dc0b51e{SSL,[ssl, http/1.1]}{127.0.0.1:50366}
[junit4] 2> 1202684 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.e.j.s.Server Started @1204830ms
[junit4] 2> 1202684 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_DA1F49C466B7CE27-001/tempDir-001/jetty4, solrconfig=solrconfig.xml, hostContext=/, hostPort=50366, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_DA1F49C466B7CE27-001/shard-4-001/cores}
[junit4] 2> 1202684 ERROR (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1202684 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 8.0.0
[junit4] 2> 1202684 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ St
[...truncated too long message...]
ctor.java:56)
[junit4] 2> at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:348)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$1.lambda$process$1(SolrZkClient.java:269)
[junit4] 2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
[junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 1225110 INFO (zkCallback-1900-thread-1-processing-n:127.0.0.1:50247_) [n:127.0.0.1:50247_ c:collection1 s:shard4 r:core_node94 x:collection1_shard4_replica_n93] o.a.s.c.SolrCore [collection1_shard4_replica_n93] CLOSING SolrCore org.apache.solr.core.SolrCore@5fd13155
[junit4] 2> 1225110 INFO (zkCallback-1900-thread-1-processing-n:127.0.0.1:50247_) [n:127.0.0.1:50247_ c:collection1 s:shard4 r:core_node94 x:collection1_shard4_replica_n93] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1.shard4.replica_n93, tag=1607545173
[junit4] 2> 1225110 INFO (zkCallback-1900-thread-1-processing-n:127.0.0.1:50247_) [n:127.0.0.1:50247_ c:collection1 s:shard4 r:core_node94 x:collection1_shard4_replica_n93] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@560f2f33: rootName = null, domain = solr.core.collection1.shard4.replica_n93, service url = null, agent id = null] for registry solr.core.collection1.shard4.replica_n93 / com.codahale.metrics.MetricRegistry@23502c03
[junit4] 2> 1225119 INFO (zkCallback-1900-thread-1-processing-n:127.0.0.1:50247_) [n:127.0.0.1:50247_ c:collection1 s:shard4 r:core_node94 x:collection1_shard4_replica_n93] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.collection1.shard4.leader, tag=1607545173
[junit4] 2> 1225119 WARN (zkCallback-1900-thread-1-processing-n:127.0.0.1:50247_) [n:127.0.0.1:50247_ c:collection1 s:shard4 r:core_node94 x:collection1_shard4_replica_n93] o.a.s.c.RecoveryStrategy Stopping recovery for core=[collection1_shard4_replica_n93] coreNodeName=[core_node94]
[junit4] 2> 1225127 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@797245c9{SSL,[ssl, http/1.1]}{127.0.0.1:50247}
[junit4] 2> 1225128 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@46935e70{/,null,UNAVAILABLE}
[junit4] 2> 1225128 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.e.j.s.session Stopped scavenging
[junit4] 2> 1225129 ERROR (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 1225130 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[DA1F49C466B7CE27]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:35256 35256
[junit4] 2> 1225299 INFO (Thread-20147) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:35256 35256
[junit4] 2> 1225299 WARN (Thread-20147) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 14 /solr/aliases.json
[junit4] 2> 12 /solr/security.json
[junit4] 2> 12 /solr/configs/conf1
[junit4] 2> 12 /solr/collections/collection1/terms/shard2
[junit4] 2> 9 /solr/collections/collection1/terms/shard3
[junit4] 2> 7 /solr/collections/collection1/terms/shard4
[junit4] 2> 2 /solr/collections/collection1/terms/shard1
[junit4] 2> 2 /solr/collections/control_collection/terms/shard1
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 14 /solr/clusterprops.json
[junit4] 2> 14 /solr/clusterstate.json
[junit4] 2> 10 /solr/collections/collection1/state.json
[junit4] 2> 7 /solr/autoscaling.json
[junit4] 2> 4 /solr/overseer_elect/election/73384374890856467-127.0.0.1:50366_-n_0000000004
[junit4] 2> 3 /solr/collections/collection1/leader_elect/shard4/election/73384374890856464-core_node86-n_0000000000
[junit4] 2> 2 /solr/collections/collection1/leader_elect/shard2/election/73384374890856458-core_node82-n_0000000000
[junit4] 2> 2 /solr/collections/collection1/leader_elect/shard3/election/73384374890856461-core_node84-n_0000000000
[junit4] 2> 2 /solr/overseer_elect/election/73384374890856482-127.0.0.1:37320_-n_0000000009
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 14 /solr/live_nodes
[junit4] 2> 14 /solr/collections
[junit4] 2> 7 /solr/overseer/queue
[junit4] 2> 7 /solr/autoscaling/events/.auto_add_replicas
[junit4] 2> 7 /solr/overseer/collection-queue-work
[junit4] 2> 7 /solr/overseer/queue-work
[junit4] 2>
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=ChaosMonkeySafeLeaderTest -Dtests.method=test -Dtests.seed=DA1F49C466B7CE27 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=mk-MK -Dtests.timezone=America/Phoenix -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
[junit4] FAILURE 30.6s J1 | ChaosMonkeySafeLeaderTest.test <<<
[junit4] > Throwable #1: java.lang.AssertionError: expected:<0> but was:<1>
[junit4] > at __randomizedtesting.SeedInfo.seed([DA1F49C466B7CE27:524B761EC84BA3DF]:0)
[junit4] > at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test(ChaosMonkeySafeLeaderTest.java:146)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_DA1F49C466B7CE27-001
[junit4] 2> Mar 02, 2018 12:19:54 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {rnd_b=Lucene50(blocksize=128), a_t=PostingsFormat(name=MockRandom), id=Lucene50(blocksize=128)}, docValues:{_version_=DocValuesFormat(name=Lucene70), a_i=DocValuesFormat(name=Lucene70)}, maxPointsInLeafNode=1539, maxMBSortInHeap=7.971109547167098, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@739543c), locale=mk-MK, timezone=America/Phoenix
[junit4] 2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=236543344,total=529006592
[junit4] 2> NOTE: All tests run in this JVM: [ReplicationFactorTest, TestCloudNestedDocsSort, VMParamsZkACLAndCredentialsProvidersTest, TestQueryUtils, DocValuesNotIndexedTest, TestCloudRecovery, PolyFieldTest, TestHdfsCloudBackupRestore, TolerantUpdateProcessorTest, SpatialRPTFieldTypeTest, PathHierarchyTokenizerFactoryTest, TestNonDefinedSimilarityFactory, DeleteNodeTest, FastVectorHighlighterTest, LeaderInitiatedRecoveryOnShardRestartTest, TermsComponentTest, FieldMutatingUpdateProcessorTest, TestFastLRUCache, TestRequestForwarding, TestRTGBase, TestRecoveryHdfs, TestJettySolrRunner, CSVRequestHandlerTest, TestSearchPerf, TestFieldCacheWithThreads, TestSimpleTextCodec, TestLeaderElectionWithEmptyReplica, TestFieldCacheSort, TestXmlQParserPlugin, TestXIncludeConfig, JavabinLoaderTest, TestLockTree, DistributedFacetExistsSmallTest, TestCustomStream, BlockJoinFacetRandomTest, TestTrie, AlternateDirectoryTest, CheckHdfsIndexTest, HdfsAutoAddReplicasIntegrationTest, CustomCollectionTest, DistributedQueryComponentCustomSortTest, UtilsToolTest, ResponseBuilderTest, QueryEqualityTest, BlobRepositoryCloudTest, HdfsSyncSliceTest, HttpPartitionOnCommitTest, TestPerFieldSimilarity, TestBackupRepositoryFactory, TestManagedSchema, SuggesterFSTTest, MoveReplicaTest, DistributedFacetPivotSmallAdvancedTest, ParsingFieldUpdateProcessorsTest, BasicAuthStandaloneTest, TestRecovery, TestNumericRangeQuery32, TestStressUserVersions, QueryElevationComponentTest, TestPrepRecovery, TestStressRecovery, TestRandomFlRTGCloud, TestSortableTextField, DistributedIntervalFacetingTest, TestAuthenticationFramework, TestDistributedGrouping, CoreAdminRequestStatusTest, TestDefaultStatsCache, TestSubQueryTransformerCrossCore, SolrCLIZkUtilsTest, TestImplicitCoreProperties, TestInPlaceUpdatesStandalone, TestCloudJSONFacetJoinDomain, TemplateUpdateProcessorTest, TestCustomDocTransformer, TestSolr4Spatial, TestCustomSort, TestSolrConfigHandlerConcurrent, JsonLoaderTest, TestElisionMultitermQuery, SampleTest, XsltUpdateRequestHandlerTest, DistribJoinFromCollectionTest, TestReload, TestLegacyNumericUtils, ScriptEngineTest, AutoScalingHandlerTest, DOMUtilTest, CollectionsAPIAsyncDistributedZkTest, ChaosMonkeySafeLeaderTest]
[junit4] Completed [231/778 (1!)] on J1 in 30.91s, 1 test, 1 failure <<< FAILURES!
[...truncated 49660 lines...]